From 80ae6d9411035e35a650d06eab23b140a8309816 Mon Sep 17 00:00:00 2001 From: Christian Pointner Date: Thu, 25 Jan 2018 22:28:46 +0100 Subject: started a complete rewrite --- .gitignore | 1 + src/examples/audio.json | 2 +- src/examples/avminimal.json | 2 +- src/examples/elevate2013.json | 2 +- src/examples/lac2013.json | 2 +- src/examples/test.json | 2 +- src/examples/test2.json | 6 +- src/flufigut-client.sh | 140 ---------- src/flufigut.py | 579 +++++------------------------------------- src/manage.sh | 189 -------------- src/update.sh | 42 --- 11 files changed, 73 insertions(+), 894 deletions(-) delete mode 100755 src/flufigut-client.sh delete mode 100755 src/manage.sh delete mode 100755 src/update.sh diff --git a/.gitignore b/.gitignore index 133ee47..940b6dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ src/output src/id_rsa* +src/old diff --git a/src/examples/audio.json b/src/examples/audio.json index cdaa734..a173fae 100644 --- a/src/examples/audio.json +++ b/src/examples/audio.json @@ -20,7 +20,7 @@ "medium": { "audio": 128 }, "low": { "audio": 96 } }, - "machines": { + "workers": { "calypso": [ "input-alsa", "encoder-alsa-vorbis-160", "encoder-alsa-vorbis-128", "encoder-alsa-vorbis-96", "muxer-audio-ogg-high", "muxer-audio-ogg-medium", "muxer-audio-ogg-low", "streamer-local1" ] } diff --git a/src/examples/avminimal.json b/src/examples/avminimal.json index a3e8f76..b6c6da5 100644 --- a/src/examples/avminimal.json +++ b/src/examples/avminimal.json @@ -24,7 +24,7 @@ "bitrates": { "vp8": { "720p25": 2048, "480p25": 1024, "360p25": 600 } }, - "machines": { + "workers": { "localhost": [ "input-video", "input-audio", "encoder-video-vp8-360p25", "encoder-audio-vorbis-96", "muxer-av-webm-low", "streamer-local1" ] } }, diff --git a/src/examples/elevate2013.json b/src/examples/elevate2013.json index 607b209..487033a 100644 --- a/src/examples/elevate2013.json +++ b/src/examples/elevate2013.json @@ -41,7 +41,7 @@ "vp8": { "720p25": 1800, "480p25": 1000, "360p25": 600, "240p25": 300 }, "mjpeg": { "720p25": 95 } }, - "machines": { + "workers": { "telesto": [ "input-sdi", "resize-sdi", "resample-sdi", "input-sdi-en", "resample-sdi-en", "encoder-sdi-vorbis-160", "encoder-sdi-vorbis-128", "encoder-sdi-vorbis-96", "encoder-sdi-vorbis-64", "encoder-sdi-en-vorbis-160", "encoder-sdi-en-vorbis-128", "encoder-sdi-en-vorbis-96", "encoder-sdi-en-vorbis-64", diff --git a/src/examples/lac2013.json b/src/examples/lac2013.json index 65fe19a..45bc8fe 100644 --- a/src/examples/lac2013.json +++ b/src/examples/lac2013.json @@ -40,7 +40,7 @@ "vp8": { "720p25": 1800, "480p25": 1000, "360p25": 600, "240p25": 300 }, "mjpeg": { "720p25": 95 } }, - "machines": { + "workers": { "calypso": [ "input-sdi", "resize-sdi", "resample-sdi", "encoder-sdi-vorbis-160", "encoder-sdi-vorbis-128", "encoder-sdi-vorbis-96", "encoder-sdi-vorbis-64", "encoder-sdi-aac-160", "encoder-sdi-aac-128", "encoder-sdi-aac-96", "encoder-sdi-aac-64", diff --git a/src/examples/test.json b/src/examples/test.json index ea3faa3..9f2d189 100644 --- a/src/examples/test.json +++ b/src/examples/test.json @@ -31,7 +31,7 @@ "h264": { "720p25": 2048, "480p25": 1024, "360p25": 600 }, "vp8": { "720p25": 2048, "480p25": 1024, "360p25": 600 } }, - "machines": { + "workers": { "titan": [ "input-sdi", "resample-sdi", "resize-sdi", "encoder-sdi-vorbis-160", "encoder-sdi-vorbis-128", "encoder-sdi-vorbis-96", "encoder-sdi-aac-160", "encoder-sdi-aac-128", "encoder-sdi-aac-96", "muxer-av-flash-high", "muxer-av-flash-medium", "muxer-av-flash-low", "muxer-av-webm-high", diff --git a/src/examples/test2.json b/src/examples/test2.json index 31739b3..cb66d4d 100644 --- a/src/examples/test2.json +++ b/src/examples/test2.json @@ -31,13 +31,13 @@ "h264": { "720p25": 2048, "480p25": 1024, "360p25": 600 }, "vp8": { "720p25": 2048, "480p25": 1024, "360p25": 600 } }, - "machines": { + "workers": { "calypso": [ "input-sdi", "resize-sdi", "resample-sdi", "input-alsa", "resample-alsa", "encoder-alsa-vorbis-160", "encoder-sdi-vorbis-160", "encoder-alsa-aac-160", "encoder-alsa-aac-128", "encoder-alsa-aac-96", "encoder-sdi-aac-160", "encoder-sdi-aac-128", "encoder-sdi-aac-96", - "muxer-av-orig-flash-high", "muxer-av-orig-flash-medium", "muxer-av-orig-flash-low", - "muxer-av-en-flash-high", "muxer-av-en-flash-medium", "muxer-av-en-flash-low", + "muxer-av-orig-flash-high", "muxer-av-orig-flash-medium", "muxer-av-orig-flash-low", + "muxer-av-en-flash-high", "muxer-av-en-flash-medium", "muxer-av-en-flash-low", "muxer-audio-ogg-high", "muxer-audio-en-ogg-high", "streamer-local1" ], "telesto": [ "encoder-sdi-h264-720p25", "encoder-sdi-h264-480p25", "encoder-sdi-h264-360p25" ] } diff --git a/src/flufigut-client.sh b/src/flufigut-client.sh deleted file mode 100755 index d02b9d7..0000000 --- a/src/flufigut-client.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/sh -# -# flufigut -# -# flufigut, the flumotion configuration utility, is a simple tool -# that generates flumotion configuration files using pyhton jinja2 -# template engine and simplejson. flufigut generates planet.xml -# and worker.xml files from configuration templates and an easy to -# understand representation of the flow structure written in json. -# -# -# Copyright (C) 2012-2014 Christian Pointner -# Michael Gebetsroither -# -# This file is part of flufigut. -# -# flufigut is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# any later version. -# -# flufigut is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with flufigut. If not, see . -# - -NAME="flumotion" -LOGFILE="/var/log/flumotion/service.log" -DAEMON="/usr/sbin/$NAME" -DAEMON_ARGS="-d 3 -l $LOGFILE" -CONF_DIR="/etc/flumotion" - - -if [ "install" = "$1" ]; then - UID=`/usr/bin/id -u` - if [ $UID -ne 0 ]; then - echo "ERROR: this script must run as root for installation." - exit 1 - fi - cp $0 /usr/local/bin - getent passwd flufigut > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo -n "copy&paste public key and press enter: " - read pubkey - if [ -z "$pubkey" ]; then - echo "ERROR: no keyfile given" - exit 1 - fi - adduser flufigut --disabled-password --home /var/lib/flufigut --gecos "flufigut management" --shell "/bin/sh" - adduser flufigut flumotion - mkdir -p /var/lib/flufigut/.ssh - echo 'command="/usr/local/bin/flufigut-client.sh",no-X11-forwarding,no-agent-forwarding,no-port-forwarding '$pubkey > /var/lib/flufigut/.ssh/authorized_keys - chown -R flufigut:flufigut /var/lib/flufigut/.ssh - chmod 600 /var/lib/flufigut/.ssh/authorized_keys - echo "flufigut ALL = (flumotion) NOPASSWD: $DAEMON, /bin/tar, /bin/rm" >> /etc/sudoers - fi - echo "successfully installed flufigut client." - - exit 0 -fi - -remove_instance() { - if [ "$1" = "worker" ]; then - sudo -u flumotion rm "$CONF_DIR/workers/$2.xml" - elif [ "$1" = "manager" ]; then - sudo -u flumotion rm -rf "$CONF_DIR/managers/$2/" - sudo -u flumotion rm "$CONF_DIR/$2.passwd" - else - echo "WARN: ignorng unknown type $1" - fi -} - -start_instance() { - sudo -u flumotion $DAEMON $DAEMON_ARGS start "$1" "$2" - return "$?" -} - -stop_instance() { - sudo -u flumotion $DAEMON $DAEMON_ARGS stop "$1" "$2" - return "$?" -} - -command=`echo $SSH_ORIGINAL_COMMAND | awk '{ print $1 }'` -param=`echo $SSH_ORIGINAL_COMMAND | awk '{ print $2 }'` - -case "$command" in - install) - echo -n "install files from '$param' ... " - sudo -u flumotion tar -C "$CONF_DIR" --strip-components 1 -xzf "$param" - rm "$param" - echo "done." - ;; - remove) - echo -n "removing files ... " - while read line; do - type=`echo $line | awk '{ print $1 }'` - name=`echo $line | awk '{ print $2 }'` - remove_instance $type $name - done - echo "ok." - ;; - wipe) - echo -n "wiping files ... " - sudo -u flumotion rm "$CONF_DIR/workers/$param-"*.xml 2> /dev/null - sudo -u flumotion rm -rf "$CONF_DIR/managers/$param/" 2> /dev/null - sudo -u flumotion rm "$CONF_DIR/$param.passwd" 2> /dev/null - echo "done." - ;; - start) - echo -n "starting ... " - while read line; do - type=`echo $line | awk '{ print $1 }'` - name=`echo $line | awk '{ print $2 }'` - start_instance $type $name - done - echo "ok." - ;; - stop) - echo -n "stopping ... " - while read line; do - type=`echo $line | awk '{ print $1 }'` - name=`echo $line | awk '{ print $2 }'` - stop_instance $type $name - done - echo "ok." - ;; - scp) - $SSH_ORIGINAL_COMMAND - ;; - *) - echo "Unknown command: '$command'" - exit 1 - ;; -esac - -exit 0 diff --git a/src/flufigut.py b/src/flufigut.py index 25e86ac..42406d3 100755 --- a/src/flufigut.py +++ b/src/flufigut.py @@ -1,16 +1,15 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # flufigut # # flufigut, the flumotion configuration utility, is a simple tool # that generates flumotion configuration files using pyhton jinja2 -# template engine and simplejson. flufigut generates planet.xml -# and worker.xml files from configuration templates and an easy to -# understand representation of the flow structure written in json. +# template engine. flufigut generates planet.xml and worker.xml +# files from configuration templates and an easy to understand +# representation of the flow structure written in json or yaml. # # -# Copyright (C) 2012-2014 Christian Pointner -# Michael Gebetsroither +# Copyright (C) 2018 Christian Pointner # # This file is part of flufigut. # @@ -31,535 +30,85 @@ import string import random import sys -import simplejson as json -from exceptions import * -from jinja2 import Environment, FileSystemLoader -import shutil -import os -import crypt -import getpass +import yaml +# from jinja2 import Environment, FileSystemLoader -### helper functions ############################################ +# helper functions ############################################ # -def rand_string(size=8, chars=string.ascii_lowercase + string.ascii_uppercase + string.digits): - return ''.join(random.choice(chars) for x in range(size)) - -### parse json file ############################################# -# -if len(sys.argv) <= 2: - raise SystemExit("ERROR: No output format and or configuration file given") -output_format = sys.argv[1] +def rand_string(size=8, chars=string.ascii_lowercase + string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for x in range(size)) -cf = open(sys.argv[2], 'r') -config = json.load(cf); -cf.close(); -### initialization ############################################## +# a flufigut stream description ############################### # -globals = config['globals'] -input = config['input'] -mux = config['mux'] -stream = config['stream'] -if 'record' in config: - record = config['record'] -else: - record = {} - -if output_format == 'flumotion': - ### init flumotion configuration generator ###################### - - atmosphere = {} - flow = {} - ### sanity checks ############################################### - machines = {} - worker = {} - - for machine in globals['machines']: - for w in globals['machines'][machine]: - if w in worker: - raise SystemExit("ERROR: worker '%s' is assigned to more than one machine!!" % w) - else: - worker[w] = 0 - - ### generate input components ################################### - - - - flow['input'] = {} - - master = 0 - for source in input: - name = 'input-%s' % source - if name not in worker: - worker[name] = -1 - else: - worker[name] = 1 - flow['input'][name] = { - 'type': input[source]['type'], - 'desc': "capture raw data from %s" % (source), - 'worker': name, - 'master': input[source]['master'], - 'properties': {}, - } - if input[source]['master']: - master += 1 - properties = input[source]['properties'] - for property in properties.keys(): - if property == 'resolution': - flow['input'][name]['properties']['width'] = globals['resolutions'][properties[property]]['width'] - flow['input'][name]['properties']['height'] = globals['resolutions'][properties[property]]['height'] - flow['input'][name]['properties']['framerate'] = globals['resolutions'][properties[property]]['rate'] - else: - flow['input'][name]['properties'][property] = properties[property] - - if master == 0: - raise SystemExit("You have not configured any master clock device!") - elif master > 1: - raise SystemExit("You have configured multiple master clock devices!") - - - for mux_name in mux.keys(): - for format in mux[mux_name]['formats'].keys(): - if 'audio' in mux[mux_name]: - source = mux[mux_name]['audio'].split(':')[0] - input_samplerate = input[source]['properties']['samplerate'] - if 'samplerate' in globals['formats'][format]: - samplerate = globals['formats'][format]['samplerate'] - if samplerate != input_samplerate: - worker_name = 'resample-%s' % (source) - if worker_name not in worker: - worker[worker_name] = -1 - else: - worker[worker_name] = 1 - feeder = 'input-%s' % (mux[mux_name]['audio']) - flow['input']['resample-%s-%s' % (source, samplerate)] = { - 'type': 'audio-resample', - 'desc': "resample audio from %s to %s Hz" % (source, samplerate), - 'worker': worker_name, - 'feeder': feeder, - 'properties': { - 'samplerate': samplerate, - }, - } - - if 'video' in mux[mux_name]: - source = mux[mux_name]['video'].split(':')[0] - input_resolution = input[source]['properties']['resolution'] - for profile in mux[mux_name]['formats'][format]: - if 'video' in globals['profiles'][profile]: - if input_resolution == "": - raise SystemExit("format definition needs video but no video input given") - resolution = globals['profiles'][profile]['video'] - if input_resolution != resolution: - if globals['resolutions'][resolution]['rate'] != globals['resolutions'][input_resolution]['rate']: - raise SystemExit("ERROR: video rate conversion is not yet supported!!!") - worker_name = 'resize-%s' % (source) - if worker_name not in worker: - worker[worker_name] = -1 - else: - worker[worker_name] = 1 - feeder = 'input-%s' % (mux[mux_name]['video']) - flow['input']['resize-%s-%s' % (source, resolution)] = { - 'type': 'video-resize', - 'desc': "resize video from %s to %sx%s" % (source, globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height']), - 'worker': worker_name, - 'feeder': feeder, - 'properties': { - 'width': globals['resolutions'][resolution]['width'], - 'height': globals['resolutions'][resolution]['height'], - }, - } - - ### generate encoder and muxer components ####################### - flow['encoder_video'] = {} - flow['encoder_audio'] = {} - flow['muxer'] = {} - - - for mux_name in mux.keys(): - for format in mux[mux_name]['formats'].keys(): - for profile in mux[mux_name]['formats'][format]: - audio_encoder = 'none' - if 'audio' in mux[mux_name]: - encoder = globals['formats'][format]['audio'] - bitrate = globals['profiles'][profile]['audio'] - if 'samplerate' in globals['formats'][format]: - samplerate = globals['formats'][format]['samplerate'] - else: - samplerate = input_samplerate - - source = mux[mux_name]['audio'].split(':')[0] - input_samplerate = input[source]['properties']['samplerate'] - if samplerate != input_samplerate: - feeder = 'resample-%s-%s' % (source, samplerate) - else: - feeder = 'input-%s' % (mux[mux_name]['audio']) - audio_encoder = 'encode-%s-%s-%i-%i' % (source, encoder, bitrate, samplerate) - if audio_encoder not in flow['encoder_audio']: - worker_name = 'encoder-%s-%s-%s' % (source, encoder, bitrate) - if worker_name not in worker: - worker[worker_name] = -1 - else: - worker[worker_name] = 1 - flow['encoder_audio'][audio_encoder] = { - 'type': '%s-encode' % encoder, - 'desc': "%s encoder for %i kbit/s @ %i Hz, from %s" % (encoder, bitrate, samplerate, source), - 'worker': worker_name, - 'feeder': feeder, - 'properties': { - 'bitrate': bitrate, - }, - } - - video_encoder = 'none' - if 'video' in mux[mux_name]: - encoder = globals['formats'][format]['video'] - resolution = globals['profiles'][profile]['video'] - bitrate = globals['bitrates'][encoder][resolution] - source = mux[mux_name]['video'].split(':')[0] - input_resolution = input[source]['properties']['resolution'] - if resolution != input_resolution: - feeder = 'resize-%s-%s' % (source, resolution) - else: - feeder = 'input-%s' % (mux[mux_name]['video']) - video_encoder = 'encode-%s-%s-%s' % (source, encoder, resolution) - if video_encoder not in flow['encoder_video'].keys(): - worker_name = 'encoder-%s-%s-%s' % (source, encoder, resolution) - if worker_name not in worker: - worker[worker_name] = -1 - else: - worker[worker_name] = 1 - flow['encoder_video'][video_encoder] = { - 'type': '%s-encode' % encoder, - 'desc': "%s encoder for %sx%s, from %s" % (encoder, globals['resolutions'][resolution]['width'], globals['resolutions'][resolution]['height'], source), - 'worker': worker_name, - 'feeder': feeder, - 'properties': { - 'bitrate': bitrate, - }, - } - - muxer = globals['formats'][format]['muxer'] - worker_name = 'muxer-%s-%s-%s' % (mux_name, format, profile) - if worker_name not in worker: - worker[worker_name] = -1 - else: - worker[worker_name] = 1 - flow['muxer']['muxer-%s-%s-%s' % (mux_name, format, profile)] = { - 'type': '%s-mux' % muxer, - 'desc': "%s muxer for %s, profile %s" % (format, mux_name, profile), - 'worker': worker_name, - 'feeder_audio': audio_encoder, - 'feeder_video': video_encoder, - 'properties': {}, - } - - ### generate streamer components ################################ - flow['streamer'] = {} - flow['repeater'] = {} - - for cluster in stream.keys(): - streamer_cnt = stream[cluster]['count'] - port = stream[cluster]['port'] - localdup = None - if 'localdup' in stream[cluster].keys(): - localdup = stream[cluster]['localdup'] - for idx in range(streamer_cnt): - stream_worker = '%s%i'%(cluster, idx+1) - if stream_worker not in worker: - worker[stream_worker] = -1 - else: - worker[stream_worker] = 1 - for machine in globals['machines'].keys(): - if stream_worker in globals['machines'][machine]: - if machine in machines: - if 'porter' in machines[machine]: - if port in machines[machine]['porter']: - raise SystemExit("ERROR: porter cannot be created because machine '%s' already uses port %i" % (machine, port)) - else: - machines[machine]['porter'] = {} - else: - machines[machine] = { 'porter': {} } - - machines[machine]['porter'][port] = { - 'socket-path': "porter-%s"%(rand_string()), - 'username': rand_string(size=12), - 'password': rand_string(size=12), - } - - atmosphere['porter-%s-%i'%(machine, port)] = { - 'type': "porter", - 'desc': "Porter for %s on port %i"%(machine, port), - 'worker': stream_worker, - 'properties': { - 'port': port, - 'socket-path': machines[machine]['porter'][port]['socket-path'], - 'username': machines[machine]['porter'][port]['username'], - 'password': machines[machine]['porter'][port]['password'], - }, - } - - if localdup: - if 'porter' in machines[machine]: - if localdup['port'] in machines[machine]['porter']: - raise SystemExit("ERROR: porter cannot be created because machine '%s' already uses port %i" % (machine, localdup['port'])) - else: - machines[machine]['porter'] = {} - machines[machine]['porter'][localdup['port']] = { - 'socket-path': "porter-%s"%(rand_string()), - 'username': rand_string(size=12), - 'password': rand_string(size=12), - } - - atmosphere['porter-%s-%i'%(machine, localdup['port'])] = { - 'type': "porter", - 'desc': "Porter (local-only) for %s on port %i"%(machine, localdup['port']), - 'worker': stream_worker, - 'properties': { - 'port': localdup['port'], - 'interface': "localhost", - 'socket-path': machines[machine]['porter'][localdup['port']]['socket-path'], - 'username': machines[machine]['porter'][localdup['port']]['username'], - 'password': machines[machine]['porter'][localdup['port']]['password'], - }, - } - - for mux_name in stream[cluster]['muxes']: - for format in mux[mux_name]['formats'].keys(): - for profile in mux[mux_name]['formats'][format]: - muxer_feed = 'muxer-%s-%s-%s' % (mux_name, format, profile) - if 'repeater' in stream[cluster]: - repeater_worker = stream[cluster]['repeater'] - if repeater_worker not in worker: - worker[repeater_worker] = -1 - else: - worker[repeater_worker] = 1 - repeater_name = '%s-%s-%s-%s' % (stream[cluster]['repeater'], mux_name, format, profile) - flow['repeater'][repeater_name] = { - 'type': "repeater", - 'desc': "%s for %s %s-%s" % (stream[cluster]['repeater'], mux_name, format, profile), - 'worker': repeater_worker, - 'feeder': muxer_feed } - feeder = repeater_name - else: - feeder = muxer_feed - - if localdup and format in localdup['formats']: - repeater_name = 'localdup-%s%i-%s-%s-%s' % (cluster, idx+1, mux_name, format, profile) - flow['repeater'][repeater_name] = { - 'type': "repeater", - 'desc': "local duplicate on %s%i for %s %s-%s" % (cluster, idx+1, mux_name, format, profile), - 'worker': stream_worker, - 'feeder': feeder } - feeder = repeater_name - - name = '%s-local-%s%i-%s-%s-%s' % (stream[cluster]['type'], cluster, idx+1, mux_name, format, profile) - mount_point = '/%s-%s-%s.%s' % (mux_name, format, profile, globals['formats'][format]['muxer']) - flow['streamer'][name] = { - 'type': "%s-stream" % stream[cluster]['type'], - 'desc': "%s streamer for %s %s-%s (part %i of %s cluster, local duplicate)" % (stream[cluster]['type'], mux_name, format, profile, idx+1, cluster), - 'worker': stream_worker, - 'feeder': feeder, - 'properties': { - 'description': globals['description'], - 'type': 'slave', - 'porter-socket-path': machines[machine]['porter'][localdup['port']]['socket-path'], - 'porter-username': machines[machine]['porter'][localdup['port']]['username'], - 'porter-password': machines[machine]['porter'][localdup['port']]['password'], - 'mount-point': mount_point, - } - } - if 'burst-on-connect' in stream[cluster]['localdup'].keys(): - flow['streamer'][name]['properties']['burst-on-connect'] = 'true' - flow['streamer'][name]['properties']['burst-time'] = stream[cluster]['localdup']['burst-on-connect'] - - - name = '%s-%s%i-%s-%s-%s' % (stream[cluster]['type'], cluster, idx+1, mux_name, format, profile) - mount_point = '/%s-%s-%s.%s' % (mux_name, format, profile, globals['formats'][format]['muxer']) - if streamer_cnt > 1: - hostname = "%s.%s" % (stream[cluster]['hostname'] % (idx+1), globals['domain']) - if idx != 0: - hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (idx), globals['domain']) - else: - hostname_next = "%s.%s" % (stream[cluster]['hostname'] % (streamer_cnt), globals['domain']) +class Description: + + def __init__(self): + self.globals = {} + self.input = {} + self.mux = {} + self.stream = {} + self.record = {} + + def _sanity_check(self): + # TODO: add more sanity checks + components = {} + for _, worker in self.globals['workers'].items(): + for c in worker: + if c in components: + raise Exception("ERROR: component '%s' is assigned to more than one worker!" % c) else: - hostname = "%s.%s" % (stream[cluster]['hostname'], globals['domain']) - flow['streamer'][name] = { - 'type': "%s-stream" % stream[cluster]['type'], - 'desc': "%s streamer for %s %s-%s (part %i of %s cluster)" % (stream[cluster]['type'], mux_name, format, profile, idx+1, cluster), - 'worker': stream_worker, - 'feeder': feeder, - 'properties': { - 'description': globals['description'], - 'type': 'slave', - 'porter-socket-path': machines[machine]['porter'][port]['socket-path'], - 'porter-username': machines[machine]['porter'][port]['username'], - 'porter-password': machines[machine]['porter'][port]['password'], - 'mount-point': mount_point, - 'hostname': hostname, - 'port': port, - } - } - - if 'stats' in globals.keys(): - flow['streamer'][name]['plugs'] = {} - if 'rrd' in globals['stats'].keys(): - flow['streamer'][name]['plugs']['rrd'] = {} - flow['streamer'][name]['plugs']['rrd']['clients'] = "%s/%s_clients.rrd" % (globals['stats']['rrd']['directory'], name) - flow['streamer'][name]['plugs']['rrd']['bytes'] = "%s/%s_bytes.rrd" % (globals['stats']['rrd']['directory'], name) - - if 'sfive' in globals['stats'].keys(): - flow['streamer'][name]['plugs']['sfive'] = {} - flow['streamer'][name]['plugs']['sfive']['socket'] = globals['stats']['sfive']['socket'] - flow['streamer'][name]['plugs']['sfive']['duration'] = globals['stats']['sfive']['duration'] - flow['streamer'][name]['plugs']['sfive']['tags'] = globals['stats']['sfive']['tags'] - flow['streamer'][name]['plugs']['sfive']['hostname'] = '%s%i' % (cluster, idx+1) - flow['streamer'][name]['plugs']['sfive']['content-id'] = mux_name - flow['streamer'][name]['plugs']['sfive']['format'] = format - flow['streamer'][name]['plugs']['sfive']['quality'] = profile - - for prop in stream[cluster]: - if prop == 'max-con': - flow['streamer'][name]['properties']['client-limit'] = stream[cluster][prop] - if streamer_cnt > 1: - flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point) - if prop == 'max-bw': - flow['streamer'][name]['properties']['bandwidth-limit'] = stream[cluster][prop] - if streamer_cnt > 1: - flow['streamer'][name]['properties']['redirect-on-overflow'] = "http://%s:%i%s" % (hostname_next, port, mount_point) - if prop == 'burst-on-connect': - flow['streamer'][name]['properties']['burst-on-connect'] = 'true' - flow['streamer'][name]['properties']['burst-time'] = stream[cluster][prop] - - ### generate record components ################################ - flow['recorder'] = {} - - for recorder in record.keys(): - record_worker = 'recorder-%s'%(recorder) - if record_worker not in worker: - worker[record_worker] = -1 - else: - worker[record_worker] = 1 - - for mux_name in record[recorder]['muxes']: - format = record[recorder]['muxes'][mux_name]["format"] - profile = record[recorder]['muxes'][mux_name]["profile"] - feeder = 'muxer-%s-%s-%s' % (mux_name, format, profile) + components[c] = 1 - name = 'recorder-%s-%s-%s' % (mux_name, format, profile) - flow['recorder'][name] = { - 'type': "recorder", - 'desc': "recorder for %s %s-%s" % (mux_name, format, profile), - 'worker': record_worker, - 'feeder': feeder, - 'properties': { } - } - for prop in record[recorder]: - if prop != 'muxes': - if prop == 'filename': - flow['recorder'][name]['properties'][prop] = record[recorder][prop] - else: - flow['recorder'][name]['properties'][prop] = record[recorder][prop] + def parse(self, config_file): + cf = open(config_file, 'r') + config = yaml.load(cf) + cf.close() - ### sanity checks, cont'd ####################################### + self.globals = config['globals'] + self.input = config['input'] + self.mux = config['mux'] + self.stream = config['stream'] + if 'record' in config: + self.record = config['record'] - error = 0 - for w in worker: - if worker[w] == 0: - print "WARNING: worker '%s' is not used" % w - elif worker[w] < 0: - error = error +1 - print "ERROR: worker '%s' is not assigned to any machine" % w + return self._sanity_check() - if error != 0: - raise SystemExit("%i Errors found - not generating any configuration" % error) - ### initialize and render templates ############################# - # - - import shutil - shutil.rmtree('%s/%s' % (globals['name'], output_format), ignore_errors=True) - - env = Environment(loader=FileSystemLoader('../templates/%s/%s/' % (output_format,globals['templates'])), line_statement_prefix = '%%') - - print "generating planet '%s/%s'" % (globals['manager']['machine'], globals['name']) - template = env.get_template('planet.xml') - planet = template.render(globals=globals, atmosphere=atmosphere, flow=flow) - - dir = '%s/%s/%s/managers/%s/' % (globals['name'], output_format, globals['manager']['machine'], globals['name']) - if not os.path.exists(dir): - os.makedirs(dir) - f = open('%s/planet.xml' % (dir), 'w') - f.write(planet.encode("utf8")) - f.write('\n') - f.close() - - passwd = open('%s/%s/%s/%s.passwd' % (globals['name'], output_format, globals['manager']['machine'], globals['name']), 'w') - - port = 9000 - for w in worker: - if worker[w] > 0: - machine_name = "" - for machine in globals['machines']: - if w in globals['machines'][machine]: - machine_name = machine - break - print "generating worker '%s/%s'" % (machine_name, w) - password = rand_string(12) - template = env.get_template('worker.xml') - ports = "%i-%i" % (port, port+1) - port+=2 - workerconf = template.render(globals=globals, name=w, password=password, portrange=ports) - dir = '%s/%s/%s/workers' % (globals['name'], output_format, machine) - if not os.path.exists(dir): - os.makedirs(dir) - f = open('%s/%s-%s.xml' % (dir, globals['name'], w), 'w') - f.write(workerconf.encode("utf8")) - f.write('\n') - f.close() - salt = rand_string(6) - passwd.write("%s:%s\n" % (w, crypt.crypt(password, salt))); - - if "admin" in globals: - user = globals['admin']['username'] - password = globals['admin']['password'] - else: - user = raw_input("username of administrator (leave empty for none): ") - if user == "": - raise SystemExit("WARN: empty username (not creating admin account)") - - password = getpass.getpass("password for '%s': " % user) - password2 = getpass.getpass("retype password for '%s': " % user) - if password != password2: - raise SystemExit("WARN: passwords don't match (not creating admin account)") - - salt = rand_string(6) - passwd.write("%s:%s\n" % (user, crypt.crypt(password, salt))); - - ### end flumotion configuration generator ####################### - -elif output_format == 'player': +# a flumtion planet configuration ############################# +# - ### init stream player configuration generator ################## +class Planet: - ## tba... - print "nothing here!" + def __init__(self): + self.atmosphere = {} + self.flow = {} - ### end stream player configuration generator ################### -elif output_format == 'stats': +# Main ######################################################## +# - ### init rrd statistic script generator ######################### +if __name__ == '__main__': + import traceback + import pprint + __pp = pprint.PrettyPrinter(indent=4, width=160) - ## tba... - print "nothing here!" + if len(sys.argv) <= 1: + print("ERROR: No configuration file given") + sys.exit(-1) + config_file = sys.argv[1] - ### en rrd statistic script generator ########################### + ret = 0 + try: + desc = Description() + desc.parse(config_file) -else: - raise SystemExit("OUTPUT FORMAT: unknown format '%s'" % output_format) + except Exception as e: + print("ERROR: while running app: %s" % e) + # print(traceback.format_exc()) + sys.exit(1) -### end ######################################################### + sys.exit(ret) diff --git a/src/manage.sh b/src/manage.sh deleted file mode 100755 index e44a52e..0000000 --- a/src/manage.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/bin/sh -# -# flufigut -# -# flufigut, the flumotion configuration utility, is a simple tool -# that generates flumotion configuration files using pyhton jinja2 -# template engine and simplejson. flufigut generates planet.xml -# and worker.xml files from configuration templates and an easy to -# understand representation of the flow structure written in json. -# -# -# Copyright (C) 2012-2014 Christian Pointner -# Michael Gebetsroither -# -# This file is part of flufigut. -# -# flufigut is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# any later version. -# -# flufigut is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with flufigut. If not, see . -# - -# -# Initialization: -# - generate ssh key: -# $ ssh-keygen -f id_rsa -# - configure ssh.config -# - copy flufigut-client.sh to every machine and install it -# $ sudo ./flufigut-client.sh install -# - -OUTPUT_DIR="$1/flumotion" -SSH_KEY="$1/id_rsa" -SSH_CONFIG="$1/ssh.config" -SSH_USER="flufigut" - -deploy_all() { - for machine in "$OUTPUT_DIR"/*; do - if [ -d "$machine" ]; then - machine=`basename $machine` - echo "copying files to $machine ... " - tar -C "$OUTPUT_DIR" -czf "$OUTPUT_DIR/$machine.tar.gz" "$machine" - scp -i "$SSH_KEY" -F "$SSH_CONFIG" "$OUTPUT_DIR/$machine.tar.gz" $SSH_USER@"$machine":/tmp - ssh -i "$SSH_KEY" -F "$SSH_CONFIG" $SSH_USER@"$machine" install "/tmp/$machine.tar.gz" - fi - done -} - -clean_all() { - for machine in "$OUTPUT_DIR"/*; do - if [ -d "$machine" ]; then - machine=`basename $machine` - echo "removing files from $machine ... " - generate_instance_list $machine - ssh -i "$SSH_KEY" -F "$SSH_CONFIG" $SSH_USER@"$machine" remove < "$OUTPUT_DIR/$machine.list" - fi - done -} - -wipe_all() { - for machine in `grep "^Host" $SSH_CONFIG | awk '{ print($2) }'`; do - echo "wipeing files belonging to planet '$1' from $machine ... " - ssh -i "$SSH_KEY" -F "$SSH_CONFIG" $SSH_USER@"$machine" wipe "$1" - done -} - -generate_instance_list() { - machine=$1 - - rm -f "$OUTPUT_DIR/$machine.list" - touch "$OUTPUT_DIR/$machine.list" - if [ -d "$OUTPUT_DIR/$machine/managers" ]; then - for name in "$OUTPUT_DIR/$machine/managers/"*; do - name=`basename "$name"` - echo "manager $name" >> "$OUTPUT_DIR/$machine.list" - done - fi - for name in "$OUTPUT_DIR/$machine/workers/"*.xml; do - name=`basename "$name"` - name=${name%%.xml} - echo "worker $name" >> "$OUTPUT_DIR/$machine.list" - done -} - -start_all() { - for machine in "$OUTPUT_DIR"/*; do - if [ -d "$machine" ] && [ -d "$machine/managers" ]; then - machine=`basename $machine` - echo " $machine (manager, worker)" - generate_instance_list $machine - ssh -i "$SSH_KEY" -F "$SSH_CONFIG" $SSH_USER@"$machine" start < "$OUTPUT_DIR/$machine.list" - break - fi - done - for machine in "$OUTPUT_DIR"/*; do - if [ -d "$machine" ] && [ ! -d "$machine/managers" ]; then - machine=`basename $machine` - echo " $machine (worker)" - generate_instance_list $machine - ssh -i "$SSH_KEY" -F "$SSH_CONFIG" $SSH_USER@"$machine" start < "$OUTPUT_DIR/$machine.list" - fi - done -} - -stop_all() { - for machine in "$OUTPUT_DIR"/*; do - if [ -d "$machine" ] && [ ! -d "$machine/managers" ]; then - machine=`basename $machine` - echo " $machine (worker)" - generate_instance_list $machine - ssh -i "$SSH_KEY" -F "$SSH_CONFIG" $SSH_USER@"$machine" stop < "$OUTPUT_DIR/$machine.list" - fi - done - for machine in "$OUTPUT_DIR"/*; do - if [ -d "$machine" ] && [ -d "$machine/managers" ]; then - machine=`basename $machine` - echo " $machine (manager, worker)" - generate_instance_list $machine - ssh -i "$SSH_KEY" -F "$SSH_CONFIG" $SSH_USER@"$machine" stop < "$OUTPUT_DIR/$machine.list" - break - fi - done -} - -print_usage() { - echo "flufigut management script by Christian Pointner " - echo "" - echo "Usage $0 " - echo "" - echo "Commands:" - echo " deploy copies all configuration file to all servers and installs them" - echo " clean remove all configuration file from all servers" - echo " wipe purges all configuration files belonging the planet" - echo " start starts all instances on all servers (manager starts first)" - echo " stop stop all instances on all servers (manager stops last)" - echo " restart same as stop, start" - echo " help prints this" -} - -if [ -z "$1" ]; then - print_usage - exit 1 -fi - -case "$2" in - deploy) - echo "Deploying configuration to all machines ..." - deploy_all - ;; - clean) - echo "cleaning/removing configuration from all machines ..." - clean_all - ;; - wipe) - echo "wipeing configuration for from all machines inside ssh-config ..." - wipe_all "$1" - ;; - start) - echo "Start manager and worker on all machines ..." - start_all - ;; - stop) - echo "Stop manager and worker on all machines ..." - stop_all - ;; - restart) - echo "Restart manager and worker on all machines ..." - stop_all - sleep 1 - start_all - ;; - help) - print_usage - ;; - *) - echo "Usage $0 (deploy|clean|wipe|start|stop|restart|help)" - exit 1 - ;; -esac - -exit 0 diff --git a/src/update.sh b/src/update.sh deleted file mode 100755 index 2e0b685..0000000 --- a/src/update.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# -# flufigut -# -# flufigut, the flumotion configuration utility, is a simple tool -# that generates flumotion configuration files using pyhton jinja2 -# template engine and simplejson. flufigut generates planet.xml -# and worker.xml files from configuration templates and an easy to -# understand representation of the flow structure written in json. -# -# -# Copyright (C) 2012-2014 Christian Pointner -# Michael Gebetsroither -# -# This file is part of flufigut. -# -# flufigut is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# any later version. -# -# flufigut is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with flufigut. If not, see . -# - -if [ -z "$1" ] || [ -z "$2" ]; then - echo "Usage: $0 " - exit 1 -fi - -./manage.sh $1 stop -./manage.sh $1 clean -./flufigut.py flumotion $2 -./manage.sh $1 deploy -./manage.sh $1 start - -exit 0 -- cgit v1.2.3