path: root/src/flufigut.py

#!/usr/bin/python
#
# flufigut
#
# flufigut, the flumotion configuration utility, is a simple tool
# that generates flumotion configuration files using pyhton jinja2
# template engine and simplejson. flufigut generates planet.xml
# and worker.xml files from configuration templates and an easy to
# understand representation of the flow structure written in json.
#
#
# Copyright (C) 2012 Christian Pointner <equinox@spreadspace.org>
#                    Michael Gebetsroither <michael@mgeb.org>
#
# This file is part of flufigut.
#
# flufigut is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# any later version.
#
# flufigut is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with flufigut. If not, see <http://www.gnu.org/licenses/>.
#

import string
import random
import sys
import simplejson as json
from exceptions import *
from jinja2 import Environment, FileSystemLoader
import shutil
import os
import crypt
import getpass

### helper functions ############################################
#
def rand_string(size=8, chars=string.ascii_lowercase + string.ascii_uppercase + string.digits):
  return ''.join(random.choice(chars) for x in range(size))

### parse json file #############################################
#
if len(sys.argv) <= 2:
  raise SystemExit("ERROR: No template name and or configuration file given")

cf = open(sys.argv[2], 'r')
config = json.load(cf);
cf.close();

### initialization ##############################################
#
globals = config['globals']
input = config['input']
mux = config['mux']
stream = config['stream']
atmosphere = {}
flow = {}

### sanity checks ###############################################
machines = {}
worker = {}

for machine in globals['machines']:
  for w in globals['machines'][machine]:
    if w in worker:
      raise SystemExit("ERROR: worker '%s' is assigned to more than one machine!!" % w)
    else:
      worker[w] = 0

### generate input components ###################################
flow['input'] = {}

master = 0
for source in input:
  name = 'input-%s' % source
  if name not in worker:
    worker[name] = -1
  else:
    worker[name] = 1
  flow['input'][name] = {
    'type': input[source]['type'],
    'desc': "capture raw data from %s" % (source),
    'worker': name,
    'master': input[source]['master'],
    'properties': {},
    }
  if input[source]['master']:
    master += 1
  properties = input[source]['properties']
  for property in properties.keys():
    if property == 'resolution':
      flow['input'][name]['properties']['width'] = globals['resolutions'][properties[property]]['width']
      flow['input'][name]['properties']['height'] = globals['resolutions'][properties[property]]['height']
      flow['input'][name]['properties']['framerate'] = globals['resolutions'][properties[property]]['rate']
    else:
      flow['input'][name]['properties'][property] = properties[property]

if master == 0:
  raise SystemExit("You have not configured any master clock device!")
elif master > 1:
  raise SystemExit("You have configured multiple master clock devices!")


for mux_name in mux.keys():
  for format in mux[mux_name]['formats'].keys():
    if 'audio' in mux[mux_name]:
      source = mux[mux_name]['audio'].split(':')[0]
      input_samplerate = input[source]['properties']['samplerate']
      if 'samplerate' in globals['formats'][format]:
        samplerate = globals['formats'][format]['samplerate']
        if samplerate != input_samplerate:
          worker_name = 'resample-%s' % (source)
          if worker_name not in worker:
            worker[worker_name] = -1
          else:
            worker[worker_name] = 1
          feeder = 'input-%s' % (mux[mux_name]['audio'])
          flow['input']['resample-%s-%s' % (source, samplerate)] = {
            'type': 'audio-resample',
            'desc': "resample audio from %s to %s Hz" % (source, samplerate),
            'worker': worker_name,
            'feeder': feeder,
            'properties': {
              'samplerate': samplerate,
              },
            }

    if 'video' in mux[mux_name]:
      source = mux[mux_name]['video'].split(':')[0]
      input_resolution = input[source]['properties']['resolution']
      for profile in mux[mux_name]['formats'][format]:
        if 'video' in globals['profiles'][profile]:
          if input_resolution == "":
            raise SystemExit("format definition needs video but no video input given")
          resolution = globals['profiles'][profile]['video']
          if input_resolution != resolution:
            if globals['resolutions'][resolution]['rate'] != globals['resolutions'][input_resolution]['rate']:
              raise SystemExit("ERROR: video rate conversion is not yet supported!!!")
            worker_name = 'resize-%s' % (source)
            if worker_name not in worker:
              worker[worker_name] = -1
            else:
              worker[worker_name] = 1
            feeder = 'input-%s' % (mux[mux_name]['video'])
            flow['input']['resize-%s-%s' % (source, resolution)] = {
              'type': 'video-resize',
              'desc': "resize video from %s to %sx%s" % (source, globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height']),
              'worker': worker_name,
              'feeder': feeder,
              'properties': {
                'width': globals['resolutions'][resolution]['width'],
                'height': globals['resolutions'][resolution]['height'],
                },
              }

### generate encoder and muxer components #######################
flow['encoder_video'] = {}
flow['encoder_audio'] = {}
flow['muxer'] = {}


for mux_name in mux.keys():
  for format in mux[mux_name]['formats'].keys():
    for profile in mux[mux_name]['formats'][format]:
      audio_encoder = 'none'
      if 'audio' in mux[mux_name]:
        encoder = globals['formats'][format]['audio']
        bitrate = globals['profiles'][profile]['audio']
        if 'samplerate' in  globals['formats'][format]:
          samplerate = globals['formats'][format]['samplerate']
        else:
          samplerate = input_samplerate

        source = mux[mux_name]['audio'].split(':')[0]
        input_samplerate = input[source]['properties']['samplerate']
        if samplerate != input_samplerate:
          feeder = 'resample-%s-%s' % (source, samplerate)
        else:
          feeder = 'input-%s' % (mux[mux_name]['audio'])
        audio_encoder = 'encode-%s-%s-%i-%i' % (source, encoder, bitrate, samplerate)
        if audio_encoder not in flow['encoder_audio']:
          worker_name = 'encoder-%s-%s-%s' % (source, encoder, bitrate)
          if worker_name not in worker:
            worker[worker_name] = -1
          else:
            worker[worker_name] = 1
          flow['encoder_audio'][audio_encoder] = {
            'type': '%s-encode' % encoder,
            'desc': "%s encoder for %i kbit/s @ %i Hz, from %s" % (encoder, bitrate, samplerate, source),
            'worker': worker_name,
            'feeder': feeder,
            'properties': {
              'bitrate': bitrate,
              },
            }

      video_encoder = 'none'
      if 'video' in mux[mux_name]:
        encoder = globals['formats'][format]['video']
        resolution = globals['profiles'][profile]['video']
        bitrate = globals['bitrates'][encoder][resolution]
        source = mux[mux_name]['video'].split(':')[0]
        input_resolution = input[source]['properties']['resolution']
        if resolution != input_resolution:
          feeder = 'resize-%s-%s' % (source, resolution)
        else:
          feeder = 'input-%s' % (mux[mux_name]['video'])
        video_encoder = 'encode-%s-%s-%s' % (source, encoder, resolution)
        if video_encoder not in flow['encoder_video'].keys():
          worker_name = 'encoder-%s-%s-%s' % (source, encoder, resolution)
          if worker_name not in worker:
            worker[worker_name] = -1
          else:
            worker[worker_name] = 1
          flow['encoder_video'][video_encoder] = {
            'type': '%s-encode' % encoder,
            'desc': "%s encoder for %sx%s, from %s" % (encoder, globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height'], source),
            'worker': worker_name,
            'feeder': feeder,
            'properties': {
              'bitrate': bitrate,
              },
            }

      muxer = globals['formats'][format]['muxer']
      worker_name = 'muxer-%s-%s-%s' % (mux_name, format, profile)
      if worker_name not in worker:
        worker[worker_name] = -1
      else:
        worker[worker_name] = 1
      flow['muxer']['muxer-%s-%s-%s' % (mux_name, format, profile)] = {
        'type': '%s-mux' % muxer,
        'desc': "%s muxer for %s, profile %s" % (format, mux_name, profile),
        'worker': worker_name,
        'feeder_audio': audio_encoder,
        'feeder_video': video_encoder,
        'properties': {},
        }


### generate streamer components ################################
flow['streamer'] = {}

for cluster in stream.keys():
  streamer_cnt = stream[cluster]['count']
  port = stream[cluster]['port']
  for idx in range(streamer_cnt):
    stream_worker = '%s%i'%(cluster, idx+1)
    for machine in globals['machines'].keys():
      if stream_worker in globals['machines'][machine]:
        if machine in machines:
          if 'porter' in machines[machine]:
            if port in machines[machine]['porter']:
              raise SystemExit("ERROR: porter cannot be created because machine '%s' already uses port %i" % (machine, port))
          else:
            machines[machine]['porter'] = {}
        else:
          machines[machine] = { 'porter': {} }

        machines[machine]['porter'][port] = {
          'socket-path': "porter-%s"%(rand_string()),
          'username': rand_string(size=12),
          'password': rand_string(size=12),
          }

        if stream_worker not in worker:
          worker[stream_worker] = -1
        else:
          worker[stream_worker] = 1
        atmosphere['porter-%s-%i'%(machine, port)] = {
          'type': "porter",
          'desc': "Porter for %s on port %i"%(machine, port),
          'worker': stream_worker,
          'properties': {
            'port': port,
            'socket-path': machines[machine]['porter'][port]['socket-path'],
            'username': machines[machine]['porter'][port]['username'],
            'password': machines[machine]['porter'][port]['password'],
            },
          }

        for mux_name in stream[cluster]['muxes']:
          for format in mux[mux_name]['formats'].keys():
            for profile in mux[mux_name]['formats'][format]:
              feeder = 'muxer-%s-%s-%s' % (mux_name, format, profile)
              name = '%s-%s%i-%s-%s-%s' % (stream[cluster]['type'], cluster, idx+1, mux_name, format, profile)
              mount_point =  '/%s-%s-%s.%s' % (mux_name, format, profile, globals['formats'][format]['muxer'])
              if streamer_cnt > 1:
                hostname = "%s.%s" % (stream[cluster]['hostname'] % (idx+1), globals['domain'])
                if idx != 0:
                  hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (idx), globals['domain'])
                else:
                  hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (streamer_cnt), globals['domain'])
              else:
                hostname = "%s.%s" % (stream[cluster]['hostname'], globals['domain'])
              flow['streamer'][name] = {
                'type': "%s-stream" % stream[cluster]['type'],
                'desc': "%s streamer for %s %s-%s (part %i of %s cluster)" % (stream[cluster]['type'], mux_name, format, profile, idx+1, cluster),
                'worker': stream_worker,
                'feeder': feeder,
                'rrd_clients' : "%s/%s_clients.rrd" % (globals['rrd-dir'], name),
                'rrd_bytes' : "%s/%s_bytes.rrd" % (globals['rrd-dir'], name),
                'properties': {
                  'description': globals['description'],
                  'type': 'slave',
                  'porter-socket-path': machines[machine]['porter'][port]['socket-path'],
                  'porter-username': machines[machine]['porter'][port]['username'],
                  'porter-password': machines[machine]['porter'][port]['password'],
                  'mount-point': mount_point,
                  'hostname': hostname,
                  'port': port,
                  }
                }
              for prop in stream[cluster]:
                if prop == 'max-con':
                  flow['streamer'][name]['properties']['client-limit'] = stream[cluster][prop]
                  if streamer_cnt > 1:
                    flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point)
                if prop == 'max-bw':
                  flow['streamer'][name]['properties']['bandwidth-limit'] = stream[cluster][prop]
                  if streamer_cnt > 1:
                    flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point)
                if prop == 'burst-on-connect':
                  flow['streamer'][name]['properties']['burst-on-connect'] = 'true'
                  flow['streamer'][name]['properties']['burst-time'] = stream[cluster][prop]

### sanity checks, cont'd #######################################

error = 0
for w in worker:
  if worker[w] == 0:
    print "WARNING: worker '%s' is not used" % w
  elif worker[w] < 0:
    error = error +1
    print "ERROR: worker '%s' is not assigned to any machine" % w

if error != 0:
  raise SystemExit("%i Errors found - not generating any configuration" % error)

### initialize and render templates #############################
#

import shutil
shutil.rmtree('output', ignore_errors=True)

env = Environment(loader=FileSystemLoader('../templates/%s/' % (sys.argv[1])), line_statement_prefix = '%%')

print "generating planet '%s/%s'" % (globals['manager']['machine'], globals['name'])
template = env.get_template('planet.xml')
planet = template.render(globals=globals, atmosphere=atmosphere, flow=flow)

dir = 'output/%s/managers/%s/' % (globals['manager']['machine'], globals['name'])
if not os.path.exists(dir):
  os.makedirs(dir)
f = open('%s/planet.xml' % (dir), 'w')
f.write(planet.encode("utf8"))
f.write('\n')
f.close()

passwd = open('output/%s/%s.passwd' % (globals['manager']['machine'], globals['name']), 'w')

port = 9000
for w in worker:
  if worker[w] > 0:
    machine_name = ""
    for machine in globals['machines']:
      if w in globals['machines'][machine]:
        machine_name = machine
        break
    print "generating worker '%s/%s'" % (machine_name, w)
    password = rand_string(12)
    template = env.get_template('worker.xml')
    ports = "%i-%i" % (port, port+1)
    port+=2
    workerconf = template.render(globals=globals, name=w, password=password, portrange=ports)
    dir = 'output/%s/workers' % machine
    if not os.path.exists(dir):
      os.makedirs(dir)
    f = open('%s/%s-%s.xml' % (dir, globals['name'], w), 'w')
    f.write(workerconf.encode("utf8"))
    f.write('\n')
    f.close()
    salt = rand_string(6)
    passwd.write("%s:%s\n" % (w, crypt.crypt(password, salt)));

if "admin" in globals:
  user = globals['admin']['username']
  password = globals['admin']['password']
else:
  user = raw_input("username of administrator (leave empty for none): ")
  if user == "":
    raise SystemExit("WARN: empty username (not creating admin account)")

  password = getpass.getpass("password for '%s': " % user)
  password2 = getpass.getpass("retype password for '%s': " % user)
  if password != password2:
    raise SystemExit("WARN: passwords don't match (not creating admin account)")

salt = rand_string(6)
passwd.write("%s:%s\n" % (user, crypt.crypt(password, salt)));

### end #########################################################