#!/usr/bin/python # # flufigut # # flufigut, the flumotion configuration utility, is a simple tool # that generates flumotion configuration files using pyhton jinja2 # template engine and simplejson. flufigut generates planet.xml # and worker.xml files from configuration templates and an easy to # understand representation of the flow structure written in json. # # # Copyright (C) 2012 Christian Pointner # Michael Gebetsroither # # This file is part of flufigut. # # flufigut is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 2 of the License, or # any later version. # # flufigut is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with flufigut. If not, see . # import string import random import sys import simplejson as json from exceptions import * from jinja2 import Environment, FileSystemLoader import shutil import os import crypt import getpass ### helper functions ############################################ # def rand_string(size=8, chars=string.ascii_lowercase + string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for x in range(size)) ### parse json file ############################################# # if len(sys.argv) <= 2: raise SystemExit("ERROR: No template name and or configuration file given") cf = open(sys.argv[2], 'r') config = json.load(cf); cf.close(); ### initialization ############################################## # globals = config['globals'] input = config['input'] mux = config['mux'] stream = config['stream'] atmosphere = {} flow = {} ### sanity checks ############################################### machines = {} worker = {} for machine in globals['machines']: for w in globals['machines'][machine]: if w in worker: raise SystemExit("ERROR: worker '%s' is assigned to more than one machine!!" % w) else: worker[w] = 0 ### generate input components ################################### flow['input'] = {} master = 0 for source in input: name = 'input-%s' % source if name not in worker: worker[name] = -1 else: worker[name] = 1 flow['input'][name] = { 'type': input[source]['type'], 'desc': "capture raw data from %s" % (source), 'worker': name, 'master': input[source]['master'], 'properties': {}, } if input[source]['master']: master += 1 properties = input[source]['properties'] for property in properties.keys(): if property == 'resolution': flow['input'][name]['properties']['width'] = globals['resolutions'][properties[property]]['width'] flow['input'][name]['properties']['height'] = globals['resolutions'][properties[property]]['height'] flow['input'][name]['properties']['framerate'] = globals['resolutions'][properties[property]]['rate'] else: flow['input'][name]['properties'][property] = properties[property] if master == 0: raise SystemExit("You have not configured any master clock device!") elif master > 1: raise SystemExit("You have configured multiple master clock devices!") samplerates = [] for mux_name in mux.keys(): for format in mux[mux_name]['formats'].keys(): if 'audio' in mux[mux_name]: source = mux[mux_name]['audio'].split(':')[0] input_samplerate = input[source]['properties']['samplerate'] if 'samplerate' in globals['formats'][format]: samplerate = globals['formats'][format]['samplerate'] if samplerate not in samplerates: if samplerate != input_samplerate: worker_name = 'resample-%s' % (source) if worker_name not in worker: worker[worker_name] = -1 else: worker[worker_name] = 1 feeder = 'input-%s' % (mux[mux_name]['audio']) flow['input']['resample-%s-%s' % (source, samplerate)] = { 'type': 'audio-resample', 'desc': "resample audio from %s to %s Hz" % (source, samplerate), 'worker': 'resample', 'feeder': feeder, 'properties': { 'samplerate': samplerate, }, } if 'video' in mux[mux_name]: source = mux[mux_name]['video'].split(':')[0] input_resolution = input[source]['properties']['resolution'] for profile in mux[mux_name]['formats'][format]: if 'video' in globals['profiles'][profile]: if input_resolution == "": raise SystemExit("format definition needs video but no video input given") resolution = globals['profiles'][profile]['video'] if input_resolution != resolution: if globals['resolutions'][resolution]['rate'] != globals['resolutions'][input_resolution]['rate']: raise SystemExit("ERROR: video rate conversion is not yet supported!!!") worker_name = 'resize-%s' % (source) if worker_name not in worker: worker[worker_name] = -1 else: worker[worker_name] = 1 feeder = 'input-%s' % (mux[mux_name]['video']) flow['input']['resize-%s-%s' % (source, resolution)] = { 'type': 'video-resize', 'desc': "resize video from %s to %sx%s" % (source, globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height']), 'worker': 'resize', 'feeder': feeder, 'properties': { 'width': globals['resolutions'][resolution]['width'], 'height': globals['resolutions'][resolution]['height'], }, } ### generate encoder and muxer components ####################### flow['encoder_video'] = {} flow['encoder_audio'] = {} flow['muxer'] = {} # for format in transcode.keys(): # for profile in transcode[format]: # video_encoder = 'none' # if 'video' in globals['formats'][format]: # encoder = globals['formats'][format]['video'] # resolution = globals['profiles'][profile]['video'] # bitrate = globals['bitrates'][encoder][resolution] # if resolution != input_resolution: # feeder = 'resize-%s' % resolution # else: # if input_type == 'av': # feeder = '%s:%s' % (input_name, "video") # elif input_type == 'vo': # feeder = '%s' % (input_name) # else: # raise SystemExit("format definition needs audio but no audio input given") # video_encoder = 'encode-%s-%s' % (encoder, resolution) # if video_encoder not in flow['encoder_video'].keys(): # worker_name = 'encoder-%s-%s' % (encoder, resolution) # if worker_name not in worker: # worker[worker_name] = -1 # else: # worker[worker_name] = 1 # flow['encoder_video'][video_encoder] = { # 'type': '%s-encode' % encoder, # 'desc': "%s encoder for %sx%s" % (encoder, globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height']), # 'worker': worker_name, # 'feeder': feeder, # 'properties': { # 'bitrate': bitrate, # }, # } # audio_encoder = 'none' # if 'audio' in globals['formats'][format]: # encoder = globals['formats'][format]['audio'] # bitrate = globals['profiles'][profile]['audio'] # if 'samplerate' in globals['formats'][format]: # samplerate = globals['formats'][format]['samplerate'] # else: # samplerate = input_samplerate # if samplerate != input_samplerate: # feeder = 'resample-%s' % samplerate # else: # if input_type == 'av': # feeder = '%s:%s' % (input_name, "audio") # elif input_type == 'ao': # feeder = '%s' % (input_name) # else: # raise SystemExit("format definition needs audio but no audio input given") # audio_encoder = 'encode-%s-%i-%i' % (encoder, bitrate, samplerate) # if audio_encoder not in flow['encoder_audio']: # worker_name = 'encoder-%s-%s' % (encoder, bitrate) # if worker_name not in worker: # worker[worker_name] = -1 # else: # worker[worker_name] = 1 # flow['encoder_audio'][audio_encoder] = { # 'type': '%s-encode' % encoder, # 'desc': "%s encoder for %i kbit/s @ %i Hz" % (encoder, bitrate, samplerate), # 'worker': worker_name, # 'feeder': feeder, # 'properties': { # 'bitrate': bitrate, # }, # } # muxer = globals['formats'][format]['muxer'] # worker_name = 'muxer-%s-%s' % (format, profile) # if worker_name not in worker: # worker[worker_name] = -1 # else: # worker[worker_name] = 1 # flow['muxer']['muxer-%s-%s' % (format, profile)] = { # 'type': '%s-mux' % muxer, # 'desc': "%s muxer profile %s" % (format, profile), # 'worker': worker_name, # 'feeder_audio': audio_encoder, # 'feeder_video': video_encoder, # 'properties': {}, # } ### generate streamer components ################################ flow['streamer'] = {} # for cluster in stream.keys(): # streamer_cnt = stream[cluster]['count'] # port = stream[cluster]['port'] # for idx in range(streamer_cnt): # stream_worker = '%s%i'%(cluster, idx+1) # for machine in globals['machines'].keys(): # if stream_worker in globals['machines'][machine]: # if machine in machines: # if 'porter' in machines[machine]: # if port in machines[machine]['porter']: # raise SystemExit("ERROR: porter cannot be created because machine '%s' already uses port %i" % (machine, port)) # else: # machines[machine]['porter'] = {} # else: # machines[machine] = { 'porter': {} } # machines[machine]['porter'][port] = { # 'socket-path': "porter-%s"%(rand_string()), # 'username': rand_string(size=12), # 'password': rand_string(size=12), # } # if stream_worker not in worker: # worker[stream_worker] = -1 # else: # worker[stream_worker] = 1 # atmosphere['porter-%s-%i'%(machine, port)] = { # 'type': "porter", # 'desc': "Porter for %s on port %i"%(machine, port), # 'worker': stream_worker, # 'properties': { # 'port': port, # 'socket-path': machines[machine]['porter'][port]['socket-path'], # 'username': machines[machine]['porter'][port]['username'], # 'password': machines[machine]['porter'][port]['password'], # }, # } # for format in stream[cluster]['formats']: # for profile in transcode[format]: # feeder = 'muxer-%s-%s' % (format, profile) # name = '%s-%s%i-%s-%s' % (stream[cluster]['type'], cluster, idx+1, format, profile) # mount_point = '/%s-%s.%s' % (format, profile, globals['formats'][format]['muxer']) # if streamer_cnt > 1: # hostname = "%s.%s" % (stream[cluster]['hostname'] % (idx+1), globals['domain']) # if idx != 0: # hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (idx), globals['domain']) # else: # hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (streamer_cnt), globals['domain']) # else: # hostname = "%s.%s" % (stream[cluster]['hostname'], globals['domain']) # flow['streamer'][name] = { # 'type': "%s-stream" % stream[cluster]['type'], # 'desc': "%s streamer for %s-%s (part %i of %s cluster)" % (stream[cluster]['type'], format, profile, idx+1, cluster), # 'worker': stream_worker, # 'feeder': feeder, # 'rrd_clients' : "%s/%s_clients.rrd" % (globals['rrd-dir'], name), # 'rrd_bytes' : "%s/%s_bytes.rrd" % (globals['rrd-dir'], name), # 'properties': { # 'description': globals['description'], # 'type': 'slave', # 'porter-socket-path': machines[machine]['porter'][port]['socket-path'], # 'porter-username': machines[machine]['porter'][port]['username'], # 'porter-password': machines[machine]['porter'][port]['password'], # 'mount-point': mount_point, # 'hostname': hostname, # 'port': port, # } # } # for prop in stream[cluster]: # if prop == 'max-con': # flow['streamer'][name]['properties']['client-limit'] = stream[cluster][prop] # if streamer_cnt > 1: # flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point) # if prop == 'max-bw': # flow['streamer'][name]['properties']['bandwidth-limit'] = stream[cluster][prop] # if streamer_cnt > 1: # flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point) # if prop == 'burst-on-connect': # flow['streamer'][name]['properties']['burst-on-connect'] = 'true' # flow['streamer'][name]['properties']['burst-time'] = stream[cluster][prop] ### sanity checks, cont'd ####################################### error = 0 for w in worker: if worker[w] == 0: print "WARNING: worker '%s' is not used" % w elif worker[w] < 0: error = error +1 print "ERROR: worker '%s' is not assigned to any machine" % w if error != 0: raise SystemExit("%i Errors found - not generating any configuration" % error) ### initialize and render templates ############################# # import shutil shutil.rmtree('output', ignore_errors=True) env = Environment(loader=FileSystemLoader('../templates/%s/' % (sys.argv[1])), line_statement_prefix = '%%') print "generating planet '%s/%s'" % (globals['manager']['machine'], globals['name']) template = env.get_template('planet.xml') planet = template.render(globals=globals, atmosphere=atmosphere, flow=flow) dir = 'output/%s/managers/%s/' % (globals['manager']['machine'], globals['name']) if not os.path.exists(dir): os.makedirs(dir) f = open('%s/planet.xml' % (dir), 'w') f.write(planet.encode("utf8")) f.write('\n') f.close() passwd = open('output/%s/%s.passwd' % (globals['manager']['machine'], globals['name']), 'w') port = 9000 for w in worker: if worker[w] > 0: machine_name = "" for machine in globals['machines']: if w in globals['machines'][machine]: machine_name = machine break print "generating worker '%s/%s'" % (machine_name, w) password = rand_string(12) template = env.get_template('worker.xml') ports = "%i-%i" % (port, port+1) port+=2 workerconf = template.render(globals=globals, name=w, password=password, portrange=ports) dir = 'output/%s/workers' % machine if not os.path.exists(dir): os.makedirs(dir) f = open('%s/%s-%s.xml' % (dir, globals['name'], w), 'w') f.write(workerconf.encode("utf8")) f.write('\n') f.close() salt = rand_string(6) passwd.write("%s:%s\n" % (w, crypt.crypt(password, salt))); if "admin" in globals: user = globals['admin']['username'] password = globals['admin']['password'] else: user = raw_input("username of administrator (leave empty for none): ") if user == "": raise SystemExit("WARN: empty username (not creating admin account)") password = getpass.getpass("password for '%s': " % user) password2 = getpass.getpass("retype password for '%s': " % user) if password != password2: raise SystemExit("WARN: passwords don't match (not creating admin account)") salt = rand_string(6) passwd.write("%s:%s\n" % (user, crypt.crypt(password, salt))); ### end #########################################################