From d33715066aab8bace17bb575a1787af40f86e67a Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 29 Apr 2013 11:05:09 +0200 Subject: Restructuring Import flask app as well as new makefile and entirely new directory structure. --- scanner/.gitignore | 3 -- scanner/Makefile | 37 +++++++++++++ scanner/TreeWalker.py | 6 --- scanner/floatapp/__init__.py | 10 ++++ scanner/floatapp/app.cfg | 13 +++++ scanner/floatapp/auth.txt | 1 + scanner/floatapp/endpoints.py | 118 ++++++++++++++++++++++++++++++++++++++++++ scanner/floatapp/jsonp.py | 18 +++++++ scanner/floatapp/login.py | 53 +++++++++++++++++++ scanner/floatapp/process.py | 52 +++++++++++++++++++ scanner/main.py | 4 +- 11 files changed, 305 insertions(+), 10 deletions(-) create mode 100644 scanner/Makefile create mode 100644 scanner/floatapp/__init__.py create mode 100644 scanner/floatapp/app.cfg create mode 100644 scanner/floatapp/auth.txt create mode 100644 scanner/floatapp/endpoints.py create mode 100644 scanner/floatapp/jsonp.py create mode 100644 scanner/floatapp/login.py create mode 100644 scanner/floatapp/process.py (limited to 'scanner') diff --git a/scanner/.gitignore b/scanner/.gitignore index 837bdaf..0d20b64 100644 --- a/scanner/.gitignore +++ b/scanner/.gitignore @@ -1,4 +1 @@ -upload.sh *.pyc -cache/* -test/* diff --git a/scanner/Makefile b/scanner/Makefile new file mode 100644 index 0000000..d246020 --- /dev/null +++ b/scanner/Makefile @@ -0,0 +1,37 @@ +.PHONY: deploy scan + +include ../deployment-config.mk +include floatapp/app.cfg + +SSH_OPTS := -q -o ControlMaster=auto -o ControlPath=.ssh-deployment.sock + +scan: + @echo " SCAN $(WEB_SERVER)" + @curl "$(WEB_SERVER_URL)/scan?username=$(subst $\",,$(ADMIN_USERNAME))&password=$(subst $\",,$(ADMIN_USERNAME))" + +deploy: + @echo " SSH $(WEB_SERVER)" + @ssh $(SSH_OPTS) -Nf $(WEB_SERVER) + + @echo " RSYNC . $(WEB_SERVER):$(FLASK_PATH)" + @ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo -u $(FLASK_USER) -v" + @rsync -aizm --delete-excluded --exclude=.ssh-deployment.sock --exclude=Makefile --exclude="*.pyc" \ + --filter="P floatapp/auth.txt" --filter="P albums/" --filter="P cache/" --exclude=.gitignore --exclude="*.swp" \ + --rsh="ssh $(SSH_OPTS)" --rsync-path="sudo -n -u $(FLASK_USER) rsync" \ + . "$(WEB_SERVER):$(FLASK_PATH)" + + @echo " CHMOD 750/640 $(WEB_SERVER):$(FLASK_PATH)/*" + @ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type f -exec chmod 640 {} \;; \ + sudo find -P '$(FLASK_PATH)' \! -path '$(FLASK_PATH)/albums/*' \! -path '$(FLASK_PATH)/cache/*' -type d -exec chmod 750 {} \;; \ + sudo chmod 750 $(FLASK_PATH)/main.py" + + @echo " CHOWN $(FLASK_USER):$(HTDOCS_USER) $(WEB_SERVER):$(FLASK_PATH){,albums,cache}" + @ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chown $(FLASK_USER):$(HTDOCS_USER) '$(FLASK_PATH)' '$(FLASK_PATH)/albums' '$(FLASK_PATH)/cache'" + @echo " CHMOD 710 $(WEB_SERVER):$(FLASK_PATH)" + @ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo chmod 710 '$(FLASK_PATH)'" + + @echo " UWSGI restart $(WEB_SERVER)" + @ssh -t $(SSH_OPTS) $(WEB_SERVER) "sudo /etc/init.d/uwsgi restart" + + @echo " SSH $(WEB_SERVER)" + @ssh -O exit $(SSH_OPTS) $(WEB_SERVER) diff --git a/scanner/TreeWalker.py b/scanner/TreeWalker.py index c2dbd53..191b187 100644 --- a/scanner/TreeWalker.py +++ b/scanner/TreeWalker.py @@ -88,12 +88,6 @@ class TreeWalker: fp = open(os.path.join(self.cache_path, "all_photos.json"), 'w') json.dump(photo_list, fp, cls=PhotoAlbumEncoder) fp.close() - photo_list.reverse() - message("caching", "latest photos path list") - fp = open(os.path.join(self.cache_path, "latest_photos.json"), 'w') - json.dump(photo_list[0:27], fp, cls=PhotoAlbumEncoder) - fp.close() - def remove_stale(self): message("cleanup", "building stale list") all_cache_entries = { "all_photos.json": True, "latest_photos.json": True } diff --git a/scanner/floatapp/__init__.py b/scanner/floatapp/__init__.py new file mode 100644 index 0000000..6e04b18 --- /dev/null +++ b/scanner/floatapp/__init__.py @@ -0,0 +1,10 @@ +from flask import Flask +from flask_login import LoginManager +import os.path + +app = Flask(__name__) +app.config.from_pyfile(os.path.join(os.path.dirname(os.path.abspath(__file__)), "app.cfg")) +login_manager = LoginManager() +import login +login_manager.setup_app(app) +import endpoints diff --git a/scanner/floatapp/app.cfg b/scanner/floatapp/app.cfg new file mode 100644 index 0000000..678585d --- /dev/null +++ b/scanner/floatapp/app.cfg @@ -0,0 +1,13 @@ +ADMIN_USERNAME = "misterscanner" +ADMIN_PASSWORD = "ilovescanning" + +PHOTO_USERNAME = "photos" # The GUI currently hardcodes 'photos', so don't change this +PHOTO_PASSWORD = "myphotopassword" + +ALBUM_PATH = "/var/www/uwsgi/photofloat/albums" +ALBUM_ACCEL = "/internal-albums" +CACHE_PATH = "/var/www/uwsgi/photofloat/cache" +CACHE_ACCEL = "/internal-cache" + +SECRET_KEY = "johlahba7shahquoequ7iod0eiGhiephahve0foo2ahshaiko9nahp0Tohch" # Replace this with something big +DEBUG = False diff --git a/scanner/floatapp/auth.txt b/scanner/floatapp/auth.txt new file mode 100644 index 0000000..2b3c598 --- /dev/null +++ b/scanner/floatapp/auth.txt @@ -0,0 +1 @@ +path/to/some/place diff --git a/scanner/floatapp/endpoints.py b/scanner/floatapp/endpoints.py new file mode 100644 index 0000000..01fee88 --- /dev/null +++ b/scanner/floatapp/endpoints.py @@ -0,0 +1,118 @@ +from floatapp import app +from floatapp.login import admin_required, login_required, is_authenticated, query_is_photo_user, query_is_admin_user, photo_user, admin_user +from floatapp.jsonp import jsonp +from process import send_process +from TreeWalker import TreeWalker +from flask import Response, abort, json, request, jsonify +from flask_login import login_user, current_user +from random import shuffle +import os +from mimetypes import guess_type + +cwd = os.path.dirname(os.path.abspath(__file__)) + +@app.route("/scan") +@admin_required +def scan_photos(): + global cwd + response = send_process([ "stdbuf", "-oL", os.path.abspath(os.path.join(cwd, "../main.py")), + os.path.abspath(app.config["ALBUM_PATH"]), os.path.abspath(app.config["CACHE_PATH"]) ], + os.path.join(cwd, "scanner.pid")) + response.headers.add("X-Accel-Buffering", "no") + response.cache_control.no_cache = True + return response + +@app.route("/auth") +def login(): + success = False + if current_user.is_authenticated(): + success = True + elif query_is_photo_user(request.form) or query_is_photo_user(request.args): + success = login_user(photo_user, remember=True) + elif query_is_admin_user(request.form) or query_is_admin_user(request.args): + success = login_user(admin_user, remember=True) + if not success: + abort(403) + return "" + +def cache_base(path): + path = path.replace('/', '-').replace(' ', '_').replace('(', '').replace('&', '').replace(',', '').replace(')', '').replace('#', '').replace('[', '').replace(']', '').replace('"', '').replace("'", '').replace('_-_', '-').lower() + while path.find("--") != -1: + path = path.replace("--", "-") + while path.find("__") != -1: + path = path.replace("__", "_") + if len(path) == 0: + path = "root" + return path + +auth_list = [ ] +def read_auth_list(): + global auth_list, cwd + f = open(os.path.join(cwd, "auth.txt"), "r") + paths = [ ] + for path in f: + path = path.strip() + paths.append(path) + paths.append(cache_base(path)) + f.close() + auth_list = paths + +# TODO: Make this run via inotify +read_auth_list() + +def check_permissions(path): + if not is_authenticated(): + for auth_path in auth_list: + if path.startswith(auth_path): + abort(403) + +@app.route("/albums/") +def albums(path): + check_permissions(path) + return accel_redirect(app.config["ALBUM_ACCEL"], app.config["ALBUM_PATH"], path) + +@app.route("/cache/") +def cache(path): + check_permissions(path) + return accel_redirect(app.config["CACHE_ACCEL"], app.config["CACHE_PATH"], path) + +def accel_redirect(internal, real, relative_name): + real_path = os.path.join(real, relative_name) + internal_path = os.path.join(internal, relative_name) + if not os.path.isfile(real_path): + abort(404) + mimetype = None + types = guess_type(real_path) + if len(types) != 0: + mimetype = types[0] + response = Response(mimetype=mimetype) + response.headers.add("X-Accel-Redirect", internal_path) + response.cache_control.public = True + if mimetype == "application/json": + response.cache_control.max_age = 3600 + else: + response.cache_control.max_age = 29030400 + return response + +@app.route("/photos") +@jsonp +def photos(): + f = open(os.path.join(app.config["CACHE_PATH"], "all_photos.json"), "r") + photos = json.load(f) + f.close() + if not is_authenticated(): + def allowed(photo): + for auth_path in auth_list: + if photo.startswith(auth_path): + return False + return True + photos = [photo for photo in photos if allowed(photo)] + count = int(request.args.get("count", len(photos))) + random = request.args.get("random") == "true" + if random: + shuffle(photos) + else: + photos.reverse() + response = jsonify(photos=photos[0:count]) + response.cache_control.no_cache = True + return response diff --git a/scanner/floatapp/jsonp.py b/scanner/floatapp/jsonp.py new file mode 100644 index 0000000..cac8ad1 --- /dev/null +++ b/scanner/floatapp/jsonp.py @@ -0,0 +1,18 @@ +import json +from functools import wraps +from flask import redirect, request, current_app +import re + +jsonp_validator = re.compile("^[a-zA-Z0-9_\-.]{1,128}$") + +def jsonp(f): + """Wraps JSONified output for JSONP""" + @wraps(f) + def decorated_function(*args, **kwargs): + callback = request.args.get('callback', False) + if callback and jsonp_validator.match(callback): + content = str(callback) + '(' + str(f(*args,**kwargs).data) + ')' + return current_app.response_class(content, mimetype='application/javascript') + else: + return f(*args, **kwargs) + return decorated_function diff --git a/scanner/floatapp/login.py b/scanner/floatapp/login.py new file mode 100644 index 0000000..d68132b --- /dev/null +++ b/scanner/floatapp/login.py @@ -0,0 +1,53 @@ +from floatapp import app, login_manager +from flask import request, abort +from flask_login import current_user, UserMixin +from functools import wraps + +class User(UserMixin): + def __init__(self, id, admin=False): + self.admin = admin + self.id = id + +photo_user = User("user") +admin_user = User("admin", True) + +@login_manager.user_loader +def load_user(id): + if id == "user": + return photo_user + elif id == "admin": + return admin_user + return None + +@login_manager.unauthorized_handler +def unauthorized(): + return abort(403) + +def login_required(fn): + @wraps(fn) + def decorated_view(*args, **kwargs): + if query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated(): + return fn(*args, **kwargs) + return app.login_manager.unauthorized() + return decorated_view + +def admin_required(fn): + @wraps(fn) + def decorated_view(*args, **kwargs): + if query_is_admin_user(request.args) or (current_user.is_authenticated() and current_user.admin): + return fn(*args, **kwargs) + return app.login_manager.unauthorized() + return decorated_view + +def query_is_photo_user(query): + username = query.get("username", None) + password = query.get("password", None) + return username == app.config["PHOTO_USERNAME"] and password == app.config["PHOTO_PASSWORD"] + +def query_is_admin_user(query): + username = query.get("username", None) + password = query.get("password", None) + return username == app.config["ADMIN_USERNAME"] and password == app.config["ADMIN_PASSWORD"] + +def is_authenticated(): + return query_is_admin_user(request.args) or query_is_photo_user(request.args) or current_user.is_authenticated() diff --git a/scanner/floatapp/process.py b/scanner/floatapp/process.py new file mode 100644 index 0000000..1bd8db2 --- /dev/null +++ b/scanner/floatapp/process.py @@ -0,0 +1,52 @@ +from flask import Response +import subprocess +import os +import sys + +class ProcessWrapper(object): + def __init__(self, process, done): + self.process = process + self.done = done + def close(self): + self.done() + if self.process.returncode is not None: + return + self.process.stdout.close() + self.process.terminate() + self.process.wait() + def __iter__(self): + return self + def __del__(self): + self.close() + def next(self): + try: + data = self.process.stdout.readline() + except: + self.close() + raise StopIteration() + if data: + return data + self.close() + raise StopIteration() + +def send_process(args, pid_file): + def setup_proc(): + f = open(pid_file, "w") + f.write(str(os.getpid())) + f.close() + os.close(0) + os.dup2(1, 2) + def tear_down_proc(): + try: + os.unlink(pid_file) + except: + pass + if os.path.exists(pid_file): + f = open(pid_file, "r") + pid = f.read() + f.close() + if os.path.exists("/proc/%s/status" % pid): + return Response("Scanner is already running.\n", mimetype="text/plain") + process = subprocess.Popen(args, close_fds=True, stdout=subprocess.PIPE, preexec_fn=setup_proc) + response = ProcessWrapper(process, tear_down_proc) + return Response(response, direct_passthrough=True, mimetype="text/plain") diff --git a/scanner/main.py b/scanner/main.py index c1fbf8c..bf9fc02 100755 --- a/scanner/main.py +++ b/scanner/main.py @@ -1,8 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 from TreeWalker import TreeWalker from CachePath import message import sys +import os def main(): reload(sys) @@ -12,6 +13,7 @@ def main(): print "usage: %s ALBUM_PATH CACHE_PATH" % sys.argv[0] return try: + os.umask(022) TreeWalker(sys.argv[1], sys.argv[2]) except KeyboardInterrupt: message("keyboard", "CTRL+C pressed, quitting.") -- cgit v1.2.3-59-g8ed1b