path: root/src/flufigut.py

#!/usr/bin/python
#
# flufigut
#
# flufigut, the flumotion configuration utility, is a simple tool
# that generates flumotion configuration files using pyhton jinja2
# template engine and simplejson. flufigut generates planet.xml
# and worker.xml files from configuration templates and an easy to
# understand representation of the flow structure written in json.
#
#
# Copyright (C) 2012 Christian Pointner <equinox@spreadspace.org>
#                    Michael Gebetsroither <michael@mgeb.org>
#
# This file is part of flufigut.
#
# flufigut is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# any later version.
#
# flufigut is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with flufigut. If not, see <http://www.gnu.org/licenses/>.
#

import string
import random
import sys
import simplejson as json
from exceptions import *
from jinja2 import Environment, FileSystemLoader
import shutil
import os
import crypt
import getpass

### helper functions ############################################
#
def rand_string(size=8, chars=string.ascii_lowercase + string.ascii_uppercase + string.digits):
  return ''.join(random.choice(chars) for x in range(size))

### parse json file #############################################
#
if len(sys.argv) <= 2:
  raise SystemExit("ERROR: No template name and or configuration file given")

cf = open(sys.argv[2], 'r')
config = json.load(cf);
cf.close();

### initialization ##############################################
#
globals = config['globals']
input = config['input']
transcode = config['transcode']
stream = config['stream']
atmosphere = {}
flow = {}

### sanity checks ###############################################
machines = {}
worker = {}

for machine in globals['machines']:
  for w in globals['machines'][machine]:
    if w in worker:
      raise SystemExit("ERROR: worker '%s' is assigned to more than one machine!!" % w)
    else:
      worker[w] = 0

### generate input components ###################################
flow['input'] = {}

if 'input' not in worker:
  worker['input'] = -1
else:
  worker['input'] = 1
flow['input']['raw-input'] = {
  'type': input['source'],
  'desc': "capture raw AV from %s" % input['source'],
  'worker': 'input',
  'properties': {},
  }
for property in input.keys():
  if property == 'samplerate':
    flow['input']['raw-input']['properties'][property] = input['samplerate']
  elif property == 'resolution':
    flow['input']['raw-input']['properties']['width'] = globals['resolutions'][input[property]]['width']
    flow['input']['raw-input']['properties']['height'] = globals['resolutions'][input[property]]['height']
    flow['input']['raw-input']['properties']['framerate'] = globals['resolutions'][input[property]]['rate']
  elif property != 'source':
    flow['input']['raw-input']['properties'][property] = input[property]

samplerates = [ ]
resolutions = [ ]
for format in transcode.keys():
  if 'samplerate' in  globals['formats'][format]:
    samplerate = globals['formats'][format]['samplerate']
    if samplerate not in samplerates:
      if input['samplerate'] != samplerate:
        samplerates.append(samplerate)
  for profile in transcode[format]:
    resolution = globals['profiles'][profile]['video']
    if resolution not in resolutions:
      if input['resolution'] != resolution:
        if globals['resolutions'][resolution]['rate'] != globals['resolutions'][input['resolution']]['rate']:
          raise SystemExit("ERROR: video rate conversion is not yet supported!!!")
        resolutions.append(resolution)

for resolution in resolutions:
  if 'resize' not in worker:
    worker['resize'] = -1
  else:
    worker['resize'] = 1
  flow['input']['resize-%s' % resolution] = {
    'type': 'video-resize',
    'desc': "resize video to %sx%s" % (globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height']),
    'worker': 'resize',
    'feeder': 'raw-input:video',
    'properties': {
      'width': globals['resolutions'][resolution]['width'],
      'height': globals['resolutions'][resolution]['height'],
      },
    }

for samplerate in samplerates:
  if 'resample' not in worker:
    worker['resample'] = -1
  else:
    worker['resample'] = 1
  flow['input']['resample-%s' % samplerate] = {
    'type': 'audio-resample',
    'desc': "resample audio to %s Hz" % samplerate,
    'worker': 'resample',
    'feeder': 'raw-input:audio',
    'properties': {
      'samplerate': samplerate,
      },
    }

### generate encoder and muxer components #######################
flow['encoder_video'] = {}
flow['encoder_audio'] = {}
flow['muxer'] = {}

for format in transcode.keys():
  for profile in transcode[format]:
    video_encoder = 'none'
    if 'video' in globals['formats'][format]:
      encoder = globals['formats'][format]['video']
      resolution = globals['profiles'][profile]['video']
      bitrate = globals['bitrates'][encoder][resolution]
      if resolution == input['resolution']:
        feeder = 'raw-input:video'
      else:
        feeder = 'resize-%s' % resolution
      video_encoder = 'encode-%s-%s' % (encoder, resolution)
      if video_encoder not in flow['encoder_video'].keys():
        worker_name = 'encoder-%s-%s' % (encoder, resolution)
        if worker_name not in worker:
          worker[worker_name] = -1
        else:
          worker[worker_name] = 1
        flow['encoder_video'][video_encoder] = {
          'type': '%s-encode' % encoder,
          'desc': "%s encoder for %sx%s" % (encoder, globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height']),
          'worker': worker_name,
          'feeder': feeder,
          'properties': {
            'bitrate': bitrate,
            },
          }

    audio_encoder = 'none'
    if 'audio' in globals['formats'][format]:
      encoder = globals['formats'][format]['audio']
      bitrate = globals['profiles'][profile]['audio']
      if 'samplerate' in  globals['formats'][format]:
        samplerate = globals['formats'][format]['samplerate']
        feeder = 'resample-%s' % samplerate
      else:
        samplerate = input['samplerate']
        feeder = 'raw-input:audio'
      audio_encoder = 'encode-%s-%i-%i' % (encoder, bitrate, samplerate)
      if audio_encoder not in flow['encoder_audio']:
        worker_name = 'encoder-%s-%s' % (encoder, bitrate)
        if worker_name not in worker:
          worker[worker_name] = -1
        else:
          worker[worker_name] = 1
        flow['encoder_audio'][audio_encoder] = {
          'type': '%s-encode' % encoder,
          'desc': "%s encoder for %i kbit/s @ %i Hz" % (encoder, bitrate, samplerate),
          'worker': worker_name,
          'feeder': feeder,
          'properties': {
            'bitrate': bitrate,
            },
          }

    muxer = globals['formats'][format]['muxer']
    worker_name = 'muxer-%s-%s' % (format, profile)
    if worker_name not in worker:
      worker[worker_name] = -1
    else:
      worker[worker_name] = 1
    flow['muxer']['muxer-%s-%s' % (format, profile)] = {
      'type': '%s-mux' % muxer,
      'desc': "%s muxer profile %s" % (format, profile),
      'worker': worker_name,
      'feeder_audio': audio_encoder,
      'feeder_video': video_encoder,
      'properties': {},
      }


### generate streamer components ################################
flow['streamer'] = {}

for cluster in stream.keys():
  streamer_cnt = stream[cluster]['count']
  port = stream[cluster]['port']
  for idx in range(streamer_cnt):
    stream_worker = '%s%i'%(cluster, idx+1)
    for machine in globals['machines'].keys():
      if stream_worker in globals['machines'][machine]:
        if machine in machines:
          if 'porter' in machines[machine]:
            if port in machines[machine]['porter']:
              raise SystemExit("ERROR: porter cannot be created because machine '%s' already uses port %i" % (machine, port))
          else:
            machines[machine]['porter'] = {}
        else:
          machines[machine] = { 'porter': {} }

        machines[machine]['porter'][port] = {
          'socket-path': "porter-%s"%(rand_string()),
          'username': rand_string(size=12),
          'password': rand_string(size=12),
          }

        if stream_worker not in worker:
          worker[stream_worker] = -1
        else:
          worker[stream_worker] = 1
        atmosphere['porter-%s-%i'%(machine, port)] = {
          'type': "porter",
          'desc': "Porter for %s on port %i"%(machine, port),
          'worker': stream_worker,
          'properties': {
            'port': port,
            'socket-path': machines[machine]['porter'][port]['socket-path'],
            'username': machines[machine]['porter'][port]['username'],
            'password': machines[machine]['porter'][port]['password'],
            },
          }

        for format in stream[cluster]['formats']:
          for profile in transcode[format]:
            feeder = 'muxer-%s-%s' % (format, profile)
            name = '%s-%s%i-%s-%s' % (stream[cluster]['type'], cluster, idx+1, format, profile)
            mount_point =  '/%s-%s.%s' % (format, profile, globals['formats'][format]['muxer'])
            if streamer_cnt > 1:
              hostname = "%s.%s" % (stream[cluster]['hostname'] % (idx+1), globals['domain'])
              if idx != 0:
                hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (idx), globals['domain'])
              else:
                hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (streamer_cnt), globals['domain'])
            else:
              hostname = "%s.%s" % (stream[cluster]['hostname'], globals['domain'])
            flow['streamer'][name] = {
              'type': "%s-stream" % stream[cluster]['type'],
              'desc': "%s streamer for %s-%s (part %i of %s cluster)" % (stream[cluster]['type'], format, profile, idx+1, cluster),
              'worker': stream_worker,
              'feeder': feeder,
              'rrd_clients' : "%s/%s_clients.rrd" % (globals['rrd-dir'], name),
              'rrd_bytes' : "%s/%s_bytes.rrd" % (globals['rrd-dir'], name),
              'properties': {
                'description': globals['description'],
                'type': 'slave',
                'porter-socket-path': machines[machine]['porter'][port]['socket-path'],
                'porter-username': machines[machine]['porter'][port]['username'],
                'porter-password': machines[machine]['porter'][port]['password'],
                'mount-point': mount_point,
                'hostname': hostname,
                'port': port,
                }
              }
            for prop in stream[cluster]:
              if prop == 'max-con':
                flow['streamer'][name]['properties']['client-limit'] = stream[cluster][prop]
                if streamer_cnt > 1:
                  flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point)
              if prop == 'max-bw':
                flow['streamer'][name]['properties']['bandwidth-limit'] = stream[cluster][prop]
                if streamer_cnt > 1:
                  flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point)
              if prop == 'burst-on-connect':
                flow['streamer'][name]['properties']['burst-on-connect'] = 'true'
                flow['streamer'][name]['properties']['burst-time'] = stream[cluster][prop]

### sanity checks, cont'd #######################################

error = 0
for w in worker:
  if worker[w] == 0:
    print "WARNING: worker '%s' is not used" % w
  elif worker[w] < 0:
    error = error +1
    print "ERROR: worker '%s' is not assigned to any machine" % w

if error != 0:
  raise SystemExit("%i Errors found - not generating any configuration" % error)

### initialize and render templates #############################
#

import shutil
shutil.rmtree('output', ignore_errors=True)

env = Environment(loader=FileSystemLoader('../templates/%s/' % (sys.argv[1])), line_statement_prefix = '%%')

print "generating planet '%s/%s'" % (globals['manager']['machine'], globals['name'])
template = env.get_template('planet.xml')
planet = template.render(globals=globals, atmosphere=atmosphere, flow=flow)

dir = 'output/%s/managers/%s/' % (globals['manager']['machine'], globals['name'])
if not os.path.exists(dir):
  os.makedirs(dir)
f = open('%s/planet.xml' % (dir), 'w')
f.write(planet.encode("utf8"))
f.write('\n')
f.close()

passwd = open('output/%s/%s.passwd' % (globals['manager']['machine'], globals['name']), 'w')

port = 9000
for w in worker:
  if worker[w] > 0:
    machine_name = ""
    for machine in globals['machines']:
      if w in globals['machines'][machine]:
        machine_name = machine
        break
    print "generating worker '%s/%s'" % (machine_name, w)
    password = rand_string(12)
    template = env.get_template('worker.xml')
    ports = "%i-%i" % (port, port+1)
    port+=2
    workerconf = template.render(globals=globals, name=w, password=password, portrange=ports)
    dir = 'output/%s/workers' % machine
    if not os.path.exists(dir):
      os.makedirs(dir)
    f = open('%s/%s-%s.xml' % (dir, globals['name'], w), 'w')
    f.write(workerconf.encode("utf8"))
    f.write('\n')
    f.close()
    salt = rand_string(6)
    passwd.write("%s:%s\n" % (w, crypt.crypt(password, salt)));

user = raw_input("username of administrator: ")
if user == "":
  raise SystemExit("WARN: empty username (not creating admin account)")

password = getpass.getpass("password for '%s': " % user)
password2 = getpass.getpass("retype password for '%s': " % user)
if password != password2:
  raise SystemExit("WARN: passwords don't match (not creating admin account)")

salt = rand_string(6)
passwd.write("%s:%s\n" % (user, crypt.crypt(password, salt)));

### end #########################################################