diff options
Diffstat (limited to 'google_appengine/google/appengine/tools/dev_appserver.py')
| -rwxr-xr-x | google_appengine/google/appengine/tools/dev_appserver.py | 3542 |
1 files changed, 3542 insertions, 0 deletions
diff --git a/google_appengine/google/appengine/tools/dev_appserver.py b/google_appengine/google/appengine/tools/dev_appserver.py new file mode 100755 index 0000000..b7e5f82 --- /dev/null +++ b/google_appengine/google/appengine/tools/dev_appserver.py @@ -0,0 +1,3542 @@ +#!/usr/bin/env python +# +# Copyright 2007 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Pure-Python application server for testing applications locally. + +Given a port and the paths to a valid application directory (with an 'app.yaml' +file), the external library directory, and a relative URL to use for logins, +creates an HTTP server that can be used to test an application locally. Uses +stubs instead of actual APIs when SetupStubs() is called first. + +Example: + root_path = '/path/to/application/directory' + login_url = '/login' + port = 8080 + template_dir = '/path/to/appserver/templates' + server = dev_appserver.CreateServer(root_path, login_url, port, template_dir) + server.serve_forever() +""" + + +from google.appengine.tools import os_compat + +import __builtin__ +import BaseHTTPServer +import Cookie +import base64 +import cStringIO +import cgi +import cgitb + +try: + import distutils.util +except ImportError: + pass + +import dummy_thread +import email.Utils +import errno +import heapq +import httplib +import imp +import inspect +import itertools +import locale +import logging +import mimetools +import mimetypes +import os +import pickle +import pprint +import random +import select + +import re +import sre_compile +import sre_constants +import sre_parse + +import socket +import sys +import time +import traceback +import types +import urlparse +import urllib + +import google +from google.pyglib import gexcept + +from google.appengine.api import apiproxy_stub_map +from google.appengine.api import appinfo +from google.appengine.api import croninfo +from google.appengine.api import datastore_admin +from google.appengine.api import datastore_file_stub +from google.appengine.api import mail_stub +from google.appengine.api import urlfetch_stub +from google.appengine.api import user_service_stub +from google.appengine.api import yaml_errors +from google.appengine.api.capabilities import capability_stub +from google.appengine.api.labs.taskqueue import taskqueue_stub +from google.appengine.api.memcache import memcache_stub +from google.appengine.api.xmpp import xmpp_service_stub + +from google.appengine import dist + +from google.appengine.tools import dev_appserver_index +from google.appengine.tools import dev_appserver_login + + +PYTHON_LIB_VAR = '$PYTHON_LIB' +DEVEL_CONSOLE_PATH = PYTHON_LIB_VAR + '/google/appengine/ext/admin' + +FILE_MISSING_EXCEPTIONS = frozenset([errno.ENOENT, errno.ENOTDIR]) + +MAX_URL_LENGTH = 2047 + +HEADER_TEMPLATE = 'logging_console_header.html' +SCRIPT_TEMPLATE = 'logging_console.js' +MIDDLE_TEMPLATE = 'logging_console_middle.html' +FOOTER_TEMPLATE = 'logging_console_footer.html' + +DEFAULT_ENV = { + 'GATEWAY_INTERFACE': 'CGI/1.1', + 'AUTH_DOMAIN': 'gmail.com', + 'TZ': 'UTC', +} + +DEFAULT_SELECT_DELAY = 30.0 + +for ext, mime_type in (('.asc', 'text/plain'), + ('.diff', 'text/plain'), + ('.csv', 'text/comma-separated-values'), + ('.rss', 'application/rss+xml'), + ('.text', 'text/plain'), + ('.wbmp', 'image/vnd.wap.wbmp')): + mimetypes.add_type(mime_type, ext) + +MAX_RUNTIME_RESPONSE_SIZE = 10 << 20 + +MAX_REQUEST_SIZE = 10 * 1024 * 1024 + +API_VERSION = '1' + +SITE_PACKAGES = os.path.normcase(os.path.join(os.path.dirname(os.__file__), + 'site-packages')) + + + +class Error(Exception): + """Base-class for exceptions in this module.""" + + +class InvalidAppConfigError(Error): + """The supplied application configuration file is invalid.""" + + +class AppConfigNotFoundError(Error): + """Application configuration file not found.""" + + +class TemplatesNotLoadedError(Error): + """Templates for the debugging console were not loaded.""" + + + +def SplitURL(relative_url): + """Splits a relative URL into its path and query-string components. + + Args: + relative_url: String containing the relative URL (often starting with '/') + to split. Should be properly escaped as www-form-urlencoded data. + + Returns: + Tuple (script_name, query_string) where: + script_name: Relative URL of the script that was accessed. + query_string: String containing everything after the '?' character. + """ + (unused_scheme, unused_netloc, path, query, + unused_fragment) = urlparse.urlsplit(relative_url) + return path, query + + +def GetFullURL(server_name, server_port, relative_url): + """Returns the full, original URL used to access the relative URL. + + Args: + server_name: Name of the local host, or the value of the 'host' header + from the request. + server_port: Port on which the request was served (string or int). + relative_url: Relative URL that was accessed, including query string. + + Returns: + String containing the original URL. + """ + if str(server_port) != '80': + netloc = '%s:%s' % (server_name, server_port) + else: + netloc = server_name + return 'http://%s%s' % (netloc, relative_url) + + + +class URLDispatcher(object): + """Base-class for handling HTTP requests.""" + + def Dispatch(self, + relative_url, + path, + headers, + infile, + outfile, + base_env_dict=None): + """Dispatch and handle an HTTP request. + + base_env_dict should contain at least these CGI variables: + REQUEST_METHOD, REMOTE_ADDR, SERVER_SOFTWARE, SERVER_NAME, + SERVER_PROTOCOL, SERVER_PORT + + Args: + relative_url: String containing the URL accessed. + path: Local path of the resource that was matched; back-references will be + replaced by values matched in the relative_url. Path may be relative + or absolute, depending on the resource being served (e.g., static files + will have an absolute path; scripts will be relative). + headers: Instance of mimetools.Message with headers from the request. + infile: File-like object with input data from the request. + outfile: File-like object where output data should be written. + base_env_dict: Dictionary of CGI environment parameters if available. + Defaults to None. + + Returns: + None if request handling is complete. + Tuple (path, headers, input_file) for an internal redirect: + path: Path of URL to redirect to. + headers: Headers to send to other dispatcher. + input_file: New input to send to new dispatcher. + """ + raise NotImplementedError + + def EndRedirect(self, dispatched_output, original_output): + """Process the end of an internal redirect. + + This method is called after all subsequent dispatch requests have finished. + By default the output from the dispatched process is copied to the original. + + This will not be called on dispatchers that do not return an internal + redirect. + + Args: + dispatched_output: StringIO buffer containing the results from the + dispatched + original_output: The original output file. + """ + original_output.write(dispatched_output.read()) + + +class URLMatcher(object): + """Matches an arbitrary URL using a list of URL patterns from an application. + + Each URL pattern has an associated URLDispatcher instance and path to the + resource's location on disk. See AddURL for more details. The first pattern + that matches an inputted URL will have its associated values returned by + Match(). + """ + + def __init__(self): + """Initializer.""" + self._url_patterns = [] + + def AddURL(self, regex, dispatcher, path, requires_login, admin_only): + """Adds a URL pattern to the list of patterns. + + If the supplied regex starts with a '^' or ends with a '$' an + InvalidAppConfigError exception will be raised. Start and end symbols + and implicitly added to all regexes, meaning we assume that all regexes + consume all input from a URL. + + Args: + regex: String containing the regular expression pattern. + dispatcher: Instance of URLDispatcher that should handle requests that + match this regex. + path: Path on disk for the resource. May contain back-references like + r'\1', r'\2', etc, which will be replaced by the corresponding groups + matched by the regex if present. + requires_login: True if the user must be logged-in before accessing this + URL; False if anyone can access this URL. + admin_only: True if the user must be a logged-in administrator to + access the URL; False if anyone can access the URL. + + Raises: + TypeError: if dispatcher is not a URLDispatcher sub-class instance. + InvalidAppConfigError: if regex isn't valid. + """ + if not isinstance(dispatcher, URLDispatcher): + raise TypeError('dispatcher must be a URLDispatcher sub-class') + + if regex.startswith('^') or regex.endswith('$'): + raise InvalidAppConfigError('regex starts with "^" or ends with "$"') + + adjusted_regex = '^%s$' % regex + + try: + url_re = re.compile(adjusted_regex) + except re.error, e: + raise InvalidAppConfigError('regex invalid: %s' % e) + + match_tuple = (url_re, dispatcher, path, requires_login, admin_only) + self._url_patterns.append(match_tuple) + + def Match(self, + relative_url, + split_url=SplitURL): + """Matches a URL from a request against the list of URL patterns. + + The supplied relative_url may include the query string (i.e., the '?' + character and everything following). + + Args: + relative_url: Relative URL being accessed in a request. + split_url: Used for dependency injection. + + Returns: + Tuple (dispatcher, matched_path, requires_login, admin_only), which are + the corresponding values passed to AddURL when the matching URL pattern + was added to this matcher. The matched_path will have back-references + replaced using values matched by the URL pattern. If no match was found, + dispatcher will be None. + """ + adjusted_url, unused_query_string = split_url(relative_url) + + for url_tuple in self._url_patterns: + url_re, dispatcher, path, requires_login, admin_only = url_tuple + the_match = url_re.match(adjusted_url) + + if the_match: + adjusted_path = the_match.expand(path) + return dispatcher, adjusted_path, requires_login, admin_only + + return None, None, None, None + + def GetDispatchers(self): + """Retrieves the URLDispatcher objects that could be matched. + + Should only be used in tests. + + Returns: + A set of URLDispatcher objects. + """ + return set([url_tuple[1] for url_tuple in self._url_patterns]) + + + +class MatcherDispatcher(URLDispatcher): + """Dispatcher across multiple URLMatcher instances.""" + + def __init__(self, + login_url, + url_matchers, + get_user_info=dev_appserver_login.GetUserInfo, + login_redirect=dev_appserver_login.LoginRedirect): + """Initializer. + + Args: + login_url: Relative URL which should be used for handling user logins. + url_matchers: Sequence of URLMatcher objects. + get_user_info: Used for dependency injection. + login_redirect: Used for dependency injection. + """ + self._login_url = login_url + self._url_matchers = tuple(url_matchers) + self._get_user_info = get_user_info + self._login_redirect = login_redirect + + def Dispatch(self, + relative_url, + path, + headers, + infile, + outfile, + base_env_dict=None): + """Dispatches a request to the first matching dispatcher. + + Matchers are checked in the order they were supplied to the constructor. + If no matcher matches, a 404 error will be written to the outfile. The + path variable supplied to this method is ignored. + """ + cookies = ', '.join(headers.getheaders('cookie')) + email_addr, admin, user_id = self._get_user_info(cookies) + + for matcher in self._url_matchers: + dispatcher, matched_path, requires_login, admin_only = matcher.Match( + relative_url) + if dispatcher is None: + continue + + logging.debug('Matched "%s" to %s with path %s', + relative_url, dispatcher, matched_path) + + if (requires_login or admin_only) and not email_addr: + logging.debug('Login required, redirecting user') + self._login_redirect(self._login_url, + base_env_dict['SERVER_NAME'], + base_env_dict['SERVER_PORT'], + relative_url, + outfile) + elif admin_only and not admin: + outfile.write('Status: %d Not authorized\r\n' + '\r\n' + 'Current logged in user %s is not ' + 'authorized to view this page.' + % (httplib.FORBIDDEN, email_addr)) + else: + forward = dispatcher.Dispatch(relative_url, + matched_path, + headers, + infile, + outfile, + base_env_dict=base_env_dict) + + if forward: + new_path, new_headers, new_input = forward + logging.info('Internal redirection to %s', new_path) + new_outfile = cStringIO.StringIO() + self.Dispatch(new_path, + None, + new_headers, + new_input, + new_outfile, + dict(base_env_dict)) + new_outfile.seek(0) + dispatcher.EndRedirect(new_outfile, outfile) + + return + + outfile.write('Status: %d URL did not match\r\n' + '\r\n' + 'Not found error: %s did not match any patterns ' + 'in application configuration.' + % (httplib.NOT_FOUND, relative_url)) + + + +class ApplicationLoggingHandler(logging.Handler): + """Python Logging handler that displays the debugging console to users.""" + + _COOKIE_NAME = '_ah_severity' + + _TEMPLATES_INITIALIZED = False + _HEADER = None + _SCRIPT = None + _MIDDLE = None + _FOOTER = None + + @staticmethod + def InitializeTemplates(header, script, middle, footer): + """Initializes the templates used to render the debugging console. + + This method must be called before any ApplicationLoggingHandler instances + are created. + + Args: + header: The header template that is printed first. + script: The script template that is printed after the logging messages. + middle: The middle element that's printed before the footer. + footer; The last element that's printed at the end of the document. + """ + ApplicationLoggingHandler._HEADER = header + ApplicationLoggingHandler._SCRIPT = script + ApplicationLoggingHandler._MIDDLE = middle + ApplicationLoggingHandler._FOOTER = footer + ApplicationLoggingHandler._TEMPLATES_INITIALIZED = True + + @staticmethod + def AreTemplatesInitialized(): + """Returns True if InitializeTemplates has been called, False otherwise.""" + return ApplicationLoggingHandler._TEMPLATES_INITIALIZED + + def __init__(self, *args, **kwargs): + """Initializer. + + Args: + args, kwargs: See logging.Handler. + + Raises: + TemplatesNotLoadedError exception if the InitializeTemplates method was + not called before creating this instance. + """ + if not self._TEMPLATES_INITIALIZED: + raise TemplatesNotLoadedError + + logging.Handler.__init__(self, *args, **kwargs) + self._record_list = [] + self._start_time = time.time() + + def emit(self, record): + """Called by the logging module each time the application logs a message. + + Args: + record: logging.LogRecord instance corresponding to the newly logged + message. + """ + self._record_list.append(record) + + def AddDebuggingConsole(self, relative_url, env, outfile): + """Prints an HTML debugging console to an output stream, if requested. + + Args: + relative_url: Relative URL that was accessed, including the query string. + Used to determine if the parameter 'debug' was supplied, in which case + the console will be shown. + env: Dictionary containing CGI environment variables. Checks for the + HTTP_COOKIE entry to see if the accessing user has any logging-related + cookies set. + outfile: Output stream to which the console should be written if either + a debug parameter was supplied or a logging cookie is present. + """ + unused_script_name, query_string = SplitURL(relative_url) + param_dict = cgi.parse_qs(query_string, True) + cookie_dict = Cookie.SimpleCookie(env.get('HTTP_COOKIE', '')) + if 'debug' not in param_dict and self._COOKIE_NAME not in cookie_dict: + return + + outfile.write(self._HEADER) + for record in self._record_list: + self._PrintRecord(record, outfile) + + outfile.write(self._MIDDLE) + outfile.write(self._SCRIPT) + outfile.write(self._FOOTER) + + def _PrintRecord(self, record, outfile): + """Prints a single logging record to an output stream. + + Args: + record: logging.LogRecord instance to print. + outfile: Output stream to which the LogRecord should be printed. + """ + message = cgi.escape(record.getMessage()) + level_name = logging.getLevelName(record.levelno).lower() + level_letter = level_name[:1].upper() + time_diff = record.created - self._start_time + outfile.write('<span class="_ah_logline_%s">\n' % level_name) + outfile.write('<span class="_ah_logline_%s_prefix">%2.5f %s ></span>\n' + % (level_name, time_diff, level_letter)) + outfile.write('%s\n' % message) + outfile.write('</span>\n') + + +_IGNORE_REQUEST_HEADERS = frozenset(['content-type', 'content-length', + 'accept-encoding', 'transfer-encoding']) + + +def SetupEnvironment(cgi_path, + relative_url, + headers, + infile, + split_url=SplitURL, + get_user_info=dev_appserver_login.GetUserInfo): + """Sets up environment variables for a CGI. + + Args: + cgi_path: Full file-system path to the CGI being executed. + relative_url: Relative URL used to access the CGI. + headers: Instance of mimetools.Message containing request headers. + infile: File-like object with input data from the request. + split_url, get_user_info: Used for dependency injection. + + Returns: + Dictionary containing CGI environment variables. + """ + env = DEFAULT_ENV.copy() + + script_name, query_string = split_url(relative_url) + + env['SCRIPT_NAME'] = '' + env['QUERY_STRING'] = query_string + env['PATH_INFO'] = urllib.unquote(script_name) + env['PATH_TRANSLATED'] = cgi_path + env['CONTENT_TYPE'] = headers.getheader('content-type', + 'application/x-www-form-urlencoded') + env['CONTENT_LENGTH'] = headers.getheader('content-length', '') + + cookies = ', '.join(headers.getheaders('cookie')) + email_addr, admin, user_id = get_user_info(cookies) + env['USER_EMAIL'] = email_addr + env['USER_ID'] = user_id + if admin: + env['USER_IS_ADMIN'] = '1' + + for key in headers: + if key in _IGNORE_REQUEST_HEADERS: + continue + adjusted_name = key.replace('-', '_').upper() + env['HTTP_' + adjusted_name] = ', '.join(headers.getheaders(key)) + + PAYLOAD_HEADER = 'HTTP_X_APPENGINE_DEVELOPMENT_PAYLOAD' + if PAYLOAD_HEADER in env: + del env[PAYLOAD_HEADER] + new_data = base64.standard_b64decode(infile.getvalue()) + infile.seek(0) + infile.truncate() + infile.write(new_data) + infile.seek(0) + env['CONTENT_LENGTH'] = str(len(new_data)) + + return env + + +def NotImplementedFake(*args, **kwargs): + """Fake for methods/functions that are not implemented in the production + environment. + """ + raise NotImplementedError('This class/method is not available.') + + +class NotImplementedFakeClass(object): + """Fake class for classes that are not implemented in the production env. + """ + __init__ = NotImplementedFake + + +def IsEncodingsModule(module_name): + """Determines if the supplied module is related to encodings in any way. + + Encodings-related modules cannot be reloaded, so they need to be treated + specially when sys.modules is modified in any way. + + Args: + module_name: Absolute name of the module regardless of how it is imported + into the local namespace (e.g., foo.bar.baz). + + Returns: + True if it's an encodings-related module; False otherwise. + """ + if (module_name in ('codecs', 'encodings') or + module_name.startswith('encodings.')): + return True + return False + + +def ClearAllButEncodingsModules(module_dict): + """Clear all modules in a module dictionary except for those modules that + are in any way related to encodings. + + Args: + module_dict: Dictionary in the form used by sys.modules. + """ + for module_name in module_dict.keys(): + if not IsEncodingsModule(module_name): + del module_dict[module_name] + + +def FakeURandom(n): + """Fake version of os.urandom.""" + bytes = '' + for _ in range(n): + bytes += chr(random.randint(0, 255)) + return bytes + + +def FakeUname(): + """Fake version of os.uname.""" + return ('Linux', '', '', '', '') + + +def FakeUnlink(path): + """Fake version of os.unlink.""" + if os.path.isdir(path): + raise OSError(errno.ENOENT, "Is a directory", path) + else: + raise OSError(errno.EPERM, "Operation not permitted", path) + + +def FakeReadlink(path): + """Fake version of os.readlink.""" + raise OSError(errno.EINVAL, "Invalid argument", path) + + +def FakeAccess(path, mode): + """Fake version of os.access where only reads are supported.""" + if not os.path.exists(path) or mode != os.R_OK: + return False + else: + return True + + +def FakeSetLocale(category, value=None, original_setlocale=locale.setlocale): + """Fake version of locale.setlocale that only supports the default.""" + if value not in (None, '', 'C', 'POSIX'): + raise locale.Error('locale emulation only supports "C" locale') + return original_setlocale(category, 'C') + + +def FakeOpen(filename, flags, mode=0777): + """Fake version of os.open.""" + raise OSError(errno.EPERM, "Operation not permitted", filename) + + +def FakeRename(src, dst): + """Fake version of os.rename.""" + raise OSError(errno.EPERM, "Operation not permitted", src) + + +def FakeUTime(path, times): + """Fake version of os.utime.""" + raise OSError(errno.EPERM, "Operation not permitted", path) + + +def FakeGetPlatform(): + """Fake distutils.util.get_platform on OS/X. Pass-through otherwise.""" + if sys.platform == 'darwin': + return 'macosx-' + else: + return distutils.util.get_platform() + + +def IsPathInSubdirectories(filename, + subdirectories, + normcase=os.path.normcase): + """Determines if a filename is contained within one of a set of directories. + + Args: + filename: Path of the file (relative or absolute). + subdirectories: Iterable collection of paths to subdirectories which the + given filename may be under. + normcase: Used for dependency injection. + + Returns: + True if the supplied filename is in one of the given sub-directories or + its hierarchy of children. False otherwise. + """ + file_dir = normcase(os.path.dirname(os.path.abspath(filename))) + for parent in subdirectories: + fixed_parent = normcase(os.path.abspath(parent)) + if os.path.commonprefix([file_dir, fixed_parent]) == fixed_parent: + return True + return False + +SHARED_MODULE_PREFIXES = set([ + 'google', + 'logging', + 'sys', + 'warnings', + + + + + 're', + 'sre_compile', + 'sre_constants', + 'sre_parse', + + + + + 'wsgiref', +]) + +NOT_SHARED_MODULE_PREFIXES = set([ + 'google.appengine.ext', +]) + + +def ModuleNameHasPrefix(module_name, prefix_set): + """Determines if a module's name belongs to a set of prefix strings. + + Args: + module_name: String containing the fully qualified module name. + prefix_set: Iterable set of module name prefixes to check against. + + Returns: + True if the module_name belongs to the prefix set or is a submodule of + any of the modules specified in the prefix_set. Otherwise False. + """ + for prefix in prefix_set: + if prefix == module_name: + return True + + if module_name.startswith(prefix + '.'): + return True + + return False + + +def SetupSharedModules(module_dict): + """Creates a module dictionary for the hardened part of the process. + + Module dictionary will contain modules that should be shared between the + hardened and unhardened parts of the process. + + Args: + module_dict: Module dictionary from which existing modules should be + pulled (usually sys.modules). + + Returns: + A new module dictionary. + """ + output_dict = {} + for module_name, module in module_dict.iteritems(): + if module is None: + continue + + if IsEncodingsModule(module_name): + output_dict[module_name] = module + continue + + shared_prefix = ModuleNameHasPrefix(module_name, SHARED_MODULE_PREFIXES) + banned_prefix = ModuleNameHasPrefix(module_name, NOT_SHARED_MODULE_PREFIXES) + + if shared_prefix and not banned_prefix: + output_dict[module_name] = module + + return output_dict + + +def GeneratePythonPaths(*p): + """Generate all valid filenames for the given file. + + Args: + p: Positional args are the folders to the file and finally the file + without a suffix. + + Returns: + A list of strings representing the given path to a file with each valid + suffix for this python build. + """ + suffixes = imp.get_suffixes() + return [os.path.join(*p) + s for s, m, t in suffixes] + + +class FakeFile(file): + """File sub-class that enforces the security restrictions of the production + environment. + """ + + ALLOWED_MODES = frozenset(['r', 'rb', 'U', 'rU']) + + ALLOWED_FILES = set(os.path.normcase(filename) + for filename in mimetypes.knownfiles + if os.path.isfile(filename)) + + ALLOWED_DIRS = set([ + os.path.normcase(os.path.realpath(os.path.dirname(os.__file__))), + os.path.normcase(os.path.abspath(os.path.dirname(os.__file__))), + ]) + + NOT_ALLOWED_DIRS = set([ + + + + + SITE_PACKAGES, + ]) + + ALLOWED_SITE_PACKAGE_DIRS = set( + os.path.normcase(os.path.abspath(os.path.join(SITE_PACKAGES, path))) + for path in [ + + ]) + + ALLOWED_SITE_PACKAGE_FILES = set( + os.path.normcase(os.path.abspath(os.path.join( + os.path.dirname(os.__file__), 'site-packages', path))) + for path in itertools.chain(*[ + + [os.path.join('Crypto')], + GeneratePythonPaths('Crypto', '__init__'), + [os.path.join('Crypto', 'Cipher')], + GeneratePythonPaths('Crypto', 'Cipher', '__init__'), + GeneratePythonPaths('Crypto', 'Cipher', 'AES'), + GeneratePythonPaths('Crypto', 'Cipher', 'ARC2'), + GeneratePythonPaths('Crypto', 'Cipher', 'ARC4'), + GeneratePythonPaths('Crypto', 'Cipher', 'Blowfish'), + GeneratePythonPaths('Crypto', 'Cipher', 'CAST'), + GeneratePythonPaths('Crypto', 'Cipher', 'DES'), + GeneratePythonPaths('Crypto', 'Cipher', 'DES3'), + GeneratePythonPaths('Crypto', 'Cipher', 'XOR'), + [os.path.join('Crypto', 'Hash')], + GeneratePythonPaths('Crypto', 'Hash', '__init__'), + GeneratePythonPaths('Crypto', 'Hash', 'HMAC'), + os.path.join('Crypto', 'Hash', 'MD2'), + os.path.join('Crypto', 'Hash', 'MD4'), + GeneratePythonPaths('Crypto', 'Hash', 'MD5'), + GeneratePythonPaths('Crypto', 'Hash', 'SHA'), + os.path.join('Crypto', 'Hash', 'SHA256'), + os.path.join('Crypto', 'Hash', 'RIPEMD'), + [os.path.join('Crypto', 'Protocol')], + GeneratePythonPaths('Crypto', 'Protocol', '__init__'), + GeneratePythonPaths('Crypto', 'Protocol', 'AllOrNothing'), + GeneratePythonPaths('Crypto', 'Protocol', 'Chaffing'), + [os.path.join('Crypto', 'PublicKey')], + GeneratePythonPaths('Crypto', 'PublicKey', '__init__'), + GeneratePythonPaths('Crypto', 'PublicKey', 'DSA'), + GeneratePythonPaths('Crypto', 'PublicKey', 'ElGamal'), + GeneratePythonPaths('Crypto', 'PublicKey', 'RSA'), + GeneratePythonPaths('Crypto', 'PublicKey', 'pubkey'), + GeneratePythonPaths('Crypto', 'PublicKey', 'qNEW'), + [os.path.join('Crypto', 'Util')], + GeneratePythonPaths('Crypto', 'Util', '__init__'), + GeneratePythonPaths('Crypto', 'Util', 'RFC1751'), + GeneratePythonPaths('Crypto', 'Util', 'number'), + GeneratePythonPaths('Crypto', 'Util', 'randpool'), + ])) + + _original_file = file + + _root_path = None + _application_paths = None + _skip_files = None + _static_file_config_matcher = None + + _allow_skipped_files = True + + _availability_cache = {} + + @staticmethod + def SetAllowedPaths(root_path, application_paths): + """Configures which paths are allowed to be accessed. + + Must be called at least once before any file objects are created in the + hardened environment. + + Args: + root_path: Absolute path to the root of the application. + application_paths: List of additional paths that the application may + access, this must include the App Engine runtime but + not the Python library directories. + """ + FakeFile._application_paths = (set(os.path.realpath(path) + for path in application_paths) | + set(os.path.abspath(path) + for path in application_paths)) + FakeFile._application_paths.add(root_path) + + FakeFile._root_path = os.path.join(root_path, '') + + FakeFile._availability_cache = {} + + @staticmethod + def SetAllowSkippedFiles(allow_skipped_files): + """Configures access to files matching FakeFile._skip_files. + + Args: + allow_skipped_files: Boolean whether to allow access to skipped files + """ + FakeFile._allow_skipped_files = allow_skipped_files + FakeFile._availability_cache = {} + + @staticmethod + def SetAllowedModule(name): + """Allow the use of a module based on where it is located. + + Meant to be used by use_library() so that it has a link back into the + trusted part of the interpreter. + + Args: + name: Name of the module to allow. + """ + stream, pathname, description = imp.find_module(name) + pathname = os.path.normcase(os.path.abspath(pathname)) + if stream: + stream.close() + FakeFile.ALLOWED_FILES.add(pathname) + FakeFile.ALLOWED_FILES.add(os.path.realpath(pathname)) + else: + assert description[2] == imp.PKG_DIRECTORY + if pathname.startswith(SITE_PACKAGES): + FakeFile.ALLOWED_SITE_PACKAGE_DIRS.add(pathname) + FakeFile.ALLOWED_SITE_PACKAGE_DIRS.add(os.path.realpath(pathname)) + else: + FakeFile.ALLOWED_DIRS.add(pathname) + FakeFile.ALLOWED_DIRS.add(os.path.realpath(pathname)) + + @staticmethod + def SetSkippedFiles(skip_files): + """Sets which files in the application directory are to be ignored. + + Must be called at least once before any file objects are created in the + hardened environment. + + Must be called whenever the configuration was updated. + + Args: + skip_files: Object with .match() method (e.g. compiled regexp). + """ + FakeFile._skip_files = skip_files + FakeFile._availability_cache = {} + + @staticmethod + def SetStaticFileConfigMatcher(static_file_config_matcher): + """Sets StaticFileConfigMatcher instance for checking if a file is static. + + Must be called at least once before any file objects are created in the + hardened environment. + + Must be called whenever the configuration was updated. + + Args: + static_file_config_matcher: StaticFileConfigMatcher instance. + """ + FakeFile._static_file_config_matcher = static_file_config_matcher + FakeFile._availability_cache = {} + + @staticmethod + def IsFileAccessible(filename, normcase=os.path.normcase): + """Determines if a file's path is accessible. + + SetAllowedPaths(), SetSkippedFiles() and SetStaticFileConfigMatcher() must + be called before this method or else all file accesses will raise an error. + + Args: + filename: Path of the file to check (relative or absolute). May be a + directory, in which case access for files inside that directory will + be checked. + normcase: Used for dependency injection. + + Returns: + True if the file is accessible, False otherwise. + """ + logical_filename = normcase(os.path.abspath(filename)) + + result = FakeFile._availability_cache.get(logical_filename) + if result is None: + result = FakeFile._IsFileAccessibleNoCache(logical_filename, + normcase=normcase) + FakeFile._availability_cache[logical_filename] = result + return result + + @staticmethod + def _IsFileAccessibleNoCache(logical_filename, normcase=os.path.normcase): + """Determines if a file's path is accessible. + + This is an internal part of the IsFileAccessible implementation. + + Args: + logical_filename: Absolute path of the file to check. + normcase: Used for dependency injection. + + Returns: + True if the file is accessible, False otherwise. + """ + logical_dirfakefile = logical_filename + if os.path.isdir(logical_filename): + logical_dirfakefile = os.path.join(logical_filename, 'foo') + + if IsPathInSubdirectories(logical_dirfakefile, [FakeFile._root_path], + normcase=normcase): + relative_filename = logical_dirfakefile[len(FakeFile._root_path):] + + if (not FakeFile._allow_skipped_files and + FakeFile._skip_files.match(relative_filename)): + logging.warning('Blocking access to skipped file "%s"', + logical_filename) + return False + + if FakeFile._static_file_config_matcher.IsStaticFile(relative_filename): + logging.warning('Blocking access to static file "%s"', + logical_filename) + return False + + if logical_filename in FakeFile.ALLOWED_FILES: + return True + + if logical_filename in FakeFile.ALLOWED_SITE_PACKAGE_FILES: + return True + + if IsPathInSubdirectories(logical_dirfakefile, + FakeFile.ALLOWED_SITE_PACKAGE_DIRS, + normcase=normcase): + return True + + allowed_dirs = FakeFile._application_paths | FakeFile.ALLOWED_DIRS + if (IsPathInSubdirectories(logical_dirfakefile, + allowed_dirs, + normcase=normcase) and + not IsPathInSubdirectories(logical_dirfakefile, + FakeFile.NOT_ALLOWED_DIRS, + normcase=normcase)): + return True + + return False + + def __init__(self, filename, mode='r', bufsize=-1, **kwargs): + """Initializer. See file built-in documentation.""" + if mode not in FakeFile.ALLOWED_MODES: + raise IOError('invalid mode: %s' % mode) + + if not FakeFile.IsFileAccessible(filename): + raise IOError(errno.EACCES, 'file not accessible', filename) + + super(FakeFile, self).__init__(filename, mode, bufsize, **kwargs) + + +from google.appengine.dist import _library +_library.SetAllowedModule = FakeFile.SetAllowedModule + + +class RestrictedPathFunction(object): + """Enforces access restrictions for functions that have a file or + directory path as their first argument.""" + + _original_os = os + + def __init__(self, original_func): + """Initializer. + + Args: + original_func: Callable that takes as its first argument the path to a + file or directory on disk; all subsequent arguments may be variable. + """ + self._original_func = original_func + + def __call__(self, path, *args, **kwargs): + """Enforces access permissions for the function passed to the constructor. + """ + if not FakeFile.IsFileAccessible(path): + raise OSError(errno.EACCES, 'path not accessible', path) + + return self._original_func(path, *args, **kwargs) + + +def GetSubmoduleName(fullname): + """Determines the leaf submodule name of a full module name. + + Args: + fullname: Fully qualified module name, e.g. 'foo.bar.baz' + + Returns: + Submodule name, e.g. 'baz'. If the supplied module has no submodule (e.g., + 'stuff'), the returned value will just be that module name ('stuff'). + """ + return fullname.rsplit('.', 1)[-1] + + + +class CouldNotFindModuleError(ImportError): + """Raised when a module could not be found. + + In contrast to when a module has been found, but cannot be loaded because of + hardening restrictions. + """ + + +def Trace(func): + """Call stack logging decorator for HardenedModulesHook class. + + This decorator logs the call stack of the HardenedModulesHook class as + it executes, indenting logging messages based on the current stack depth. + + Args: + func: the function to decorate. + + Returns: + The decorated function. + """ + + def Decorate(self, *args, **kwargs): + args_to_show = [] + if args is not None: + args_to_show.extend(str(argument) for argument in args) + if kwargs is not None: + args_to_show.extend('%s=%s' % (key, value) + for key, value in kwargs.iteritems()) + + args_string = ', '.join(args_to_show) + + self.log('Entering %s(%s)', func.func_name, args_string) + self._indent_level += 1 + try: + return func(self, *args, **kwargs) + finally: + self._indent_level -= 1 + self.log('Exiting %s(%s)', func.func_name, args_string) + + return Decorate + + +class HardenedModulesHook(object): + """Meta import hook that restricts the modules used by applications to match + the production environment. + + Module controls supported: + - Disallow native/extension modules from being loaded + - Disallow built-in and/or Python-distributed modules from being loaded + - Replace modules with completely empty modules + - Override specific module attributes + - Replace one module with another + + After creation, this object should be added to the front of the sys.meta_path + list (which may need to be created). The sys.path_importer_cache dictionary + should also be cleared, to prevent loading any non-restricted modules. + + See PEP302 for more info on how this works: + http://www.python.org/dev/peps/pep-0302/ + """ + + ENABLE_LOGGING = False + + def log(self, message, *args): + """Logs an import-related message to stderr, with indentation based on + current call-stack depth. + + Args: + message: Logging format string. + args: Positional format parameters for the logging message. + """ + if HardenedModulesHook.ENABLE_LOGGING: + indent = self._indent_level * ' ' + print >>sys.stderr, indent + (message % args) + + _WHITE_LIST_C_MODULES = [ + 'AES', + 'ARC2', + 'ARC4', + 'Blowfish', + 'CAST', + 'DES', + 'DES3', + 'MD2', + 'MD4', + 'RIPEMD', + 'SHA256', + 'XOR', + + '_Crypto_Cipher__AES', + '_Crypto_Cipher__ARC2', + '_Crypto_Cipher__ARC4', + '_Crypto_Cipher__Blowfish', + '_Crypto_Cipher__CAST', + '_Crypto_Cipher__DES', + '_Crypto_Cipher__DES3', + '_Crypto_Cipher__XOR', + '_Crypto_Hash__MD2', + '_Crypto_Hash__MD4', + '_Crypto_Hash__RIPEMD', + '_Crypto_Hash__SHA256', + 'array', + 'binascii', + 'bz2', + 'cmath', + 'collections', + 'crypt', + 'cStringIO', + 'datetime', + 'errno', + 'exceptions', + 'gc', + 'itertools', + 'math', + 'md5', + 'operator', + 'posix', + 'posixpath', + 'pyexpat', + 'sha', + 'struct', + 'sys', + 'time', + 'timing', + 'unicodedata', + 'zlib', + '_ast', + '_bisect', + '_codecs', + '_codecs_cn', + '_codecs_hk', + '_codecs_iso2022', + '_codecs_jp', + '_codecs_kr', + '_codecs_tw', + '_collections', + '_csv', + '_elementtree', + '_functools', + '_hashlib', + '_heapq', + '_locale', + '_lsprof', + '_md5', + '_multibytecodec', + '_random', + '_sha', + '_sha256', + '_sha512', + '_sre', + '_struct', + '_types', + '_weakref', + '__main__', + ] + + __CRYPTO_CIPHER_ALLOWED_MODULES = [ + 'MODE_CBC', + 'MODE_CFB', + 'MODE_CTR', + 'MODE_ECB', + 'MODE_OFB', + 'block_size', + 'key_size', + 'new', + ] + _WHITE_LIST_PARTIAL_MODULES = { + 'Crypto.Cipher.AES': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.ARC2': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.Blowfish': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.CAST': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.DES': __CRYPTO_CIPHER_ALLOWED_MODULES, + 'Crypto.Cipher.DES3': __CRYPTO_CIPHER_ALLOWED_MODULES, + + 'gc': [ + 'enable', + 'disable', + 'isenabled', + 'collect', + 'get_debug', + 'set_threshold', + 'get_threshold', + 'get_count' + ], + + + + 'os': [ + 'access', + 'altsep', + 'curdir', + 'defpath', + 'devnull', + 'environ', + 'error', + 'extsep', + 'EX_NOHOST', + 'EX_NOINPUT', + 'EX_NOPERM', + 'EX_NOUSER', + 'EX_OK', + 'EX_OSERR', + 'EX_OSFILE', + 'EX_PROTOCOL', + 'EX_SOFTWARE', + 'EX_TEMPFAIL', + 'EX_UNAVAILABLE', + 'EX_USAGE', + 'F_OK', + 'getcwd', + 'getcwdu', + 'getenv', + 'listdir', + 'lstat', + 'name', + 'NGROUPS_MAX', + 'O_APPEND', + 'O_CREAT', + 'O_DIRECT', + 'O_DIRECTORY', + 'O_DSYNC', + 'O_EXCL', + 'O_LARGEFILE', + 'O_NDELAY', + 'O_NOCTTY', + 'O_NOFOLLOW', + 'O_NONBLOCK', + 'O_RDONLY', + 'O_RDWR', + 'O_RSYNC', + 'O_SYNC', + 'O_TRUNC', + 'O_WRONLY', + 'open', + 'pardir', + 'path', + 'pathsep', + 'R_OK', + 'readlink', + 'remove', + 'rename', + 'SEEK_CUR', + 'SEEK_END', + 'SEEK_SET', + 'sep', + 'stat', + 'stat_float_times', + 'stat_result', + 'strerror', + 'TMP_MAX', + 'unlink', + 'urandom', + 'utime', + 'walk', + 'WCOREDUMP', + 'WEXITSTATUS', + 'WIFEXITED', + 'WIFSIGNALED', + 'WIFSTOPPED', + 'WNOHANG', + 'WSTOPSIG', + 'WTERMSIG', + 'WUNTRACED', + 'W_OK', + 'X_OK', + ], + } + + _MODULE_OVERRIDES = { + 'locale': { + 'setlocale': FakeSetLocale, + }, + + 'os': { + 'access': FakeAccess, + 'listdir': RestrictedPathFunction(os.listdir), + + 'lstat': RestrictedPathFunction(os.stat), + 'open': FakeOpen, + 'readlink': FakeReadlink, + 'remove': FakeUnlink, + 'rename': FakeRename, + 'stat': RestrictedPathFunction(os.stat), + 'uname': FakeUname, + 'unlink': FakeUnlink, + 'urandom': FakeURandom, + 'utime': FakeUTime, + }, + + 'distutils.util': { + 'get_platform': FakeGetPlatform, + }, + } + + _ENABLED_FILE_TYPES = ( + imp.PKG_DIRECTORY, + imp.PY_SOURCE, + imp.PY_COMPILED, + imp.C_BUILTIN, + ) + + def __init__(self, + module_dict, + imp_module=imp, + os_module=os, + dummy_thread_module=dummy_thread, + pickle_module=pickle): + """Initializer. + + Args: + module_dict: Module dictionary to use for managing system modules. + Should be sys.modules. + imp_module, os_module, dummy_thread_module, pickle_module: References to + modules that exist in the dev_appserver that must be used by this class + in order to function, even if these modules have been unloaded from + sys.modules. + """ + self._module_dict = module_dict + self._imp = imp_module + self._os = os_module + self._dummy_thread = dummy_thread_module + self._pickle = pickle + self._indent_level = 0 + + @Trace + def find_module(self, fullname, path=None): + """See PEP 302.""" + if fullname in ('cPickle', 'thread'): + return self + + search_path = path + all_modules = fullname.split('.') + try: + for index, current_module in enumerate(all_modules): + current_module_fullname = '.'.join(all_modules[:index + 1]) + if (current_module_fullname == fullname and not + self.StubModuleExists(fullname)): + self.FindModuleRestricted(current_module, + current_module_fullname, + search_path) + else: + if current_module_fullname in self._module_dict: + module = self._module_dict[current_module_fullname] + else: + module = self.FindAndLoadModule(current_module, + current_module_fullname, + search_path) + + if hasattr(module, '__path__'): + search_path = module.__path__ + except CouldNotFindModuleError: + return None + + return self + + def StubModuleExists(self, name): + """Check if the named module has a stub replacement.""" + if name in sys.builtin_module_names: + name = 'py_%s' % name + if name in dist.__all__: + return True + return False + + def ImportStubModule(self, name): + """Import the stub module replacement for the specified module.""" + if name in sys.builtin_module_names: + name = 'py_%s' % name + module = __import__(dist.__name__, {}, {}, [name]) + return getattr(module, name) + + @Trace + def FixModule(self, module): + """Prunes and overrides restricted module attributes. + + Args: + module: The module to prune. This should be a new module whose attributes + reference back to the real module's __dict__ members. + """ + if module.__name__ in self._WHITE_LIST_PARTIAL_MODULES: + allowed_symbols = self._WHITE_LIST_PARTIAL_MODULES[module.__name__] + for symbol in set(module.__dict__) - set(allowed_symbols): + if not (symbol.startswith('__') and symbol.endswith('__')): + del module.__dict__[symbol] + + if module.__name__ in self._MODULE_OVERRIDES: + module.__dict__.update(self._MODULE_OVERRIDES[module.__name__]) + + @Trace + def FindModuleRestricted(self, + submodule, + submodule_fullname, + search_path): + """Locates a module while enforcing module import restrictions. + + Args: + submodule: The short name of the submodule (i.e., the last section of + the fullname; for 'foo.bar' this would be 'bar'). + submodule_fullname: The fully qualified name of the module to find (e.g., + 'foo.bar'). + search_path: List of paths to search for to find this module. Should be + None if the current sys.path should be used. + + Returns: + Tuple (source_file, pathname, description) where: + source_file: File-like object that contains the module; in the case + of packages, this will be None, which implies to look at __init__.py. + pathname: String containing the full path of the module on disk. + description: Tuple returned by imp.find_module(). + However, in the case of an import using a path hook (e.g. a zipfile), + source_file will be a PEP-302-style loader object, pathname will be None, + and description will be a tuple filled with None values. + + Raises: + ImportError exception if the requested module was found, but importing + it is disallowed. + + CouldNotFindModuleError exception if the request module could not even + be found for import. + """ + if search_path is None: + search_path = [None] + sys.path + for path_entry in search_path: + result = self.FindPathHook(submodule, submodule_fullname, path_entry) + if result is not None: + source_file, pathname, description = result + if description == (None, None, None): + return result + else: + break + else: + self.log('Could not find module "%s"', submodule_fullname) + raise CouldNotFindModuleError() + + suffix, mode, file_type = description + + if (file_type not in (self._imp.C_BUILTIN, self._imp.C_EXTENSION) and + not FakeFile.IsFileAccessible(pathname)): + error_message = 'Access to module file denied: %s' % pathname + logging.debug(error_message) + raise ImportError(error_message) + + if (file_type not in self._ENABLED_FILE_TYPES and + submodule not in self._WHITE_LIST_C_MODULES): + error_message = ('Could not import "%s": Disallowed C-extension ' + 'or built-in module' % submodule_fullname) + logging.debug(error_message) + raise ImportError(error_message) + + return source_file, pathname, description + + def FindPathHook(self, submodule, submodule_fullname, path_entry): + """Helper for FindModuleRestricted to find a module in a sys.path entry. + + Args: + submodule: + submodule_fullname: + path_entry: A single sys.path entry, or None representing the builtins. + + Returns: + Either None (if nothing was found), or a triple (source_file, path_name, + description). See the doc string for FindModuleRestricted() for the + meaning of the latter. + """ + if path_entry is None: + if submodule_fullname in sys.builtin_module_names: + try: + result = self._imp.find_module(submodule) + except ImportError: + pass + else: + source_file, pathname, description = result + suffix, mode, file_type = description + if file_type == self._imp.C_BUILTIN: + return result + return None + + + if path_entry in sys.path_importer_cache: + importer = sys.path_importer_cache[path_entry] + else: + importer = None + for hook in sys.path_hooks: + try: + importer = hook(path_entry) + break + except ImportError: + pass + sys.path_importer_cache[path_entry] = importer + + if importer is None: + try: + return self._imp.find_module(submodule, [path_entry]) + except ImportError: + pass + else: + loader = importer.find_module(submodule) + if loader is not None: + return (loader, None, (None, None, None)) + + return None + + @Trace + def LoadModuleRestricted(self, + submodule_fullname, + source_file, + pathname, + description): + """Loads a module while enforcing module import restrictions. + + As a byproduct, the new module will be added to the module dictionary. + + Args: + submodule_fullname: The fully qualified name of the module to find (e.g., + 'foo.bar'). + source_file: File-like object that contains the module's source code, + or a PEP-302-style loader object. + pathname: String containing the full path of the module on disk. + description: Tuple returned by imp.find_module(), or (None, None, None) + in case source_file is a PEP-302-style loader object. + + Returns: + The new module. + + Raises: + ImportError exception of the specified module could not be loaded for + whatever reason. + """ + if description == (None, None, None): + return source_file.load_module(submodule_fullname) + + try: + try: + return self._imp.load_module(submodule_fullname, + source_file, + pathname, + description) + except: + if submodule_fullname in self._module_dict: + del self._module_dict[submodule_fullname] + raise + + finally: + if source_file is not None: + source_file.close() + + @Trace + def FindAndLoadModule(self, + submodule, + submodule_fullname, + search_path): + """Finds and loads a module, loads it, and adds it to the module dictionary. + + Args: + submodule: Name of the module to import (e.g., baz). + submodule_fullname: Full name of the module to import (e.g., foo.bar.baz). + search_path: Path to use for searching for this submodule. For top-level + modules this should be None; otherwise it should be the __path__ + attribute from the parent package. + + Returns: + A new module instance that has been inserted into the module dictionary + supplied to __init__. + + Raises: + ImportError exception if the module could not be loaded for whatever + reason (e.g., missing, not allowed). + """ + module = self._imp.new_module(submodule_fullname) + + if submodule_fullname == 'thread': + module.__dict__.update(self._dummy_thread.__dict__) + module.__name__ = 'thread' + elif submodule_fullname == 'cPickle': + module.__dict__.update(self._pickle.__dict__) + module.__name__ = 'cPickle' + elif submodule_fullname == 'os': + module.__dict__.update(self._os.__dict__) + elif self.StubModuleExists(submodule_fullname): + module = self.ImportStubModule(submodule_fullname) + else: + source_file, pathname, description = self.FindModuleRestricted(submodule, submodule_fullname, search_path) + module = self.LoadModuleRestricted(submodule_fullname, + source_file, + pathname, + description) + + module.__loader__ = self + self.FixModule(module) + if submodule_fullname not in self._module_dict: + self._module_dict[submodule_fullname] = module + + if submodule_fullname == 'os': + os_path_name = module.path.__name__ + os_path = self.FindAndLoadModule(os_path_name, os_path_name, search_path) + self._module_dict['os.path'] = os_path + module.__dict__['path'] = os_path + + return module + + @Trace + def GetParentPackage(self, fullname): + """Retrieves the parent package of a fully qualified module name. + + Args: + fullname: Full name of the module whose parent should be retrieved (e.g., + foo.bar). + + Returns: + Module instance for the parent or None if there is no parent module. + + Raise: + ImportError exception if the module's parent could not be found. + """ + all_modules = fullname.split('.') + parent_module_fullname = '.'.join(all_modules[:-1]) + if parent_module_fullname: + if self.find_module(fullname) is None: + raise ImportError('Could not find module %s' % fullname) + + return self._module_dict[parent_module_fullname] + return None + + @Trace + def GetParentSearchPath(self, fullname): + """Determines the search path of a module's parent package. + + Args: + fullname: Full name of the module to look up (e.g., foo.bar). + + Returns: + Tuple (submodule, search_path) where: + submodule: The last portion of the module name from fullname (e.g., + if fullname is foo.bar, then this is bar). + search_path: List of paths that belong to the parent package's search + path or None if there is no parent package. + + Raises: + ImportError exception if the module or its parent could not be found. + """ + submodule = GetSubmoduleName(fullname) + parent_package = self.GetParentPackage(fullname) + search_path = None + if parent_package is not None and hasattr(parent_package, '__path__'): + search_path = parent_package.__path__ + return submodule, search_path + + @Trace + def GetModuleInfo(self, fullname): + """Determines the path on disk and the search path of a module or package. + + Args: + fullname: Full name of the module to look up (e.g., foo.bar). + + Returns: + Tuple (pathname, search_path, submodule) where: + pathname: String containing the full path of the module on disk, + or None if the module wasn't loaded from disk (e.g. from a zipfile). + search_path: List of paths that belong to the found package's search + path or None if found module is not a package. + submodule: The relative name of the submodule that's being imported. + """ + submodule, search_path = self.GetParentSearchPath(fullname) + source_file, pathname, description = self.FindModuleRestricted(submodule, fullname, search_path) + suffix, mode, file_type = description + module_search_path = None + if file_type == self._imp.PKG_DIRECTORY: + module_search_path = [pathname] + pathname = os.path.join(pathname, '__init__%spy' % os.extsep) + return pathname, module_search_path, submodule + + @Trace + def load_module(self, fullname): + """See PEP 302.""" + all_modules = fullname.split('.') + submodule = all_modules[-1] + parent_module_fullname = '.'.join(all_modules[:-1]) + search_path = None + if parent_module_fullname and parent_module_fullname in self._module_dict: + parent_module = self._module_dict[parent_module_fullname] + if hasattr(parent_module, '__path__'): + search_path = parent_module.__path__ + + return self.FindAndLoadModule(submodule, fullname, search_path) + + @Trace + def is_package(self, fullname): + """See PEP 302 extensions.""" + submodule, search_path = self.GetParentSearchPath(fullname) + source_file, pathname, description = self.FindModuleRestricted(submodule, fullname, search_path) + suffix, mode, file_type = description + if file_type == self._imp.PKG_DIRECTORY: + return True + return False + + @Trace + def get_source(self, fullname): + """See PEP 302 extensions.""" + full_path, search_path, submodule = self.GetModuleInfo(fullname) + if full_path is None: + return None + source_file = open(full_path) + try: + return source_file.read() + finally: + source_file.close() + + @Trace + def get_code(self, fullname): + """See PEP 302 extensions.""" + full_path, search_path, submodule = self.GetModuleInfo(fullname) + if full_path is None: + return None + source_file = open(full_path) + try: + source_code = source_file.read() + finally: + source_file.close() + + source_code = source_code.replace('\r\n', '\n') + if not source_code.endswith('\n'): + source_code += '\n' + + return compile(source_code, full_path, 'exec') + + + +def ModuleHasValidMainFunction(module): + """Determines if a module has a main function that takes no arguments. + + This includes functions that have arguments with defaults that are all + assigned, thus requiring no additional arguments in order to be called. + + Args: + module: A types.ModuleType instance. + + Returns: + True if the module has a valid, reusable main function; False otherwise. + """ + if hasattr(module, 'main') and type(module.main) is types.FunctionType: + arg_names, var_args, var_kwargs, default_values = inspect.getargspec( + module.main) + if len(arg_names) == 0: + return True + if default_values is not None and len(arg_names) == len(default_values): + return True + return False + + +def GetScriptModuleName(handler_path): + """Determines the fully-qualified Python module name of a script on disk. + + Args: + handler_path: CGI path stored in the application configuration (as a path + like 'foo/bar/baz.py'). May contain $PYTHON_LIB references. + + Returns: + String containing the corresponding module name (e.g., 'foo.bar.baz'). + """ + if handler_path.startswith(PYTHON_LIB_VAR + '/'): + handler_path = handler_path[len(PYTHON_LIB_VAR):] + handler_path = os.path.normpath(handler_path) + + extension_index = handler_path.rfind('.py') + if extension_index != -1: + handler_path = handler_path[:extension_index] + module_fullname = handler_path.replace(os.sep, '.') + module_fullname = module_fullname.strip('.') + module_fullname = re.sub('\.+', '.', module_fullname) + + if module_fullname.endswith('.__init__'): + module_fullname = module_fullname[:-len('.__init__')] + + return module_fullname + + +def FindMissingInitFiles(cgi_path, module_fullname, isfile=os.path.isfile): + """Determines which __init__.py files are missing from a module's parent + packages. + + Args: + cgi_path: Absolute path of the CGI module file on disk. + module_fullname: Fully qualified Python module name used to import the + cgi_path module. + isfile: Used for testing. + + Returns: + List containing the paths to the missing __init__.py files. + """ + missing_init_files = [] + + if cgi_path.endswith('.py'): + module_base = os.path.dirname(cgi_path) + else: + module_base = cgi_path + + depth_count = module_fullname.count('.') + if cgi_path.endswith('__init__.py') or not cgi_path.endswith('.py'): + depth_count += 1 + + for index in xrange(depth_count): + current_init_file = os.path.abspath( + os.path.join(module_base, '__init__.py')) + + if not isfile(current_init_file): + missing_init_files.append(current_init_file) + + module_base = os.path.abspath(os.path.join(module_base, os.pardir)) + + return missing_init_files + + +def LoadTargetModule(handler_path, + cgi_path, + import_hook, + module_dict=sys.modules): + """Loads a target CGI script by importing it as a Python module. + + If the module for the target CGI script has already been loaded before, + the new module will be loaded in its place using the same module object, + possibly overwriting existing module attributes. + + Args: + handler_path: CGI path stored in the application configuration (as a path + like 'foo/bar/baz.py'). Should not have $PYTHON_LIB references. + cgi_path: Absolute path to the CGI script file on disk. + import_hook: Instance of HardenedModulesHook to use for module loading. + module_dict: Used for dependency injection. + + Returns: + Tuple (module_fullname, script_module, module_code) where: + module_fullname: Fully qualified module name used to import the script. + script_module: The ModuleType object corresponding to the module_fullname. + If the module has not already been loaded, this will be an empty + shell of a module. + module_code: Code object (returned by compile built-in) corresponding + to the cgi_path to run. If the script_module was previously loaded + and has a main() function that can be reused, this will be None. + """ + module_fullname = GetScriptModuleName(handler_path) + script_module = module_dict.get(module_fullname) + module_code = None + if script_module is not None and ModuleHasValidMainFunction(script_module): + logging.debug('Reusing main() function of module "%s"', module_fullname) + else: + if script_module is None: + script_module = imp.new_module(module_fullname) + script_module.__loader__ = import_hook + + try: + module_code = import_hook.get_code(module_fullname) + full_path, search_path, submodule = ( + import_hook.GetModuleInfo(module_fullname)) + script_module.__file__ = full_path + if search_path is not None: + script_module.__path__ = search_path + except: + exc_type, exc_value, exc_tb = sys.exc_info() + import_error_message = str(exc_type) + if exc_value: + import_error_message += ': ' + str(exc_value) + + logging.exception('Encountered error loading module "%s": %s', + module_fullname, import_error_message) + missing_inits = FindMissingInitFiles(cgi_path, module_fullname) + if missing_inits: + logging.warning('Missing package initialization files: %s', + ', '.join(missing_inits)) + else: + logging.error('Parent package initialization files are present, ' + 'but must be broken') + + independent_load_successful = True + + if not os.path.isfile(cgi_path): + independent_load_successful = False + else: + try: + source_file = open(cgi_path) + try: + module_code = compile(source_file.read(), cgi_path, 'exec') + script_module.__file__ = cgi_path + finally: + source_file.close() + + except OSError: + independent_load_successful = False + + if not independent_load_successful: + raise exc_type, exc_value, exc_tb + + module_dict[module_fullname] = script_module + + return module_fullname, script_module, module_code + + +def ExecuteOrImportScript(handler_path, cgi_path, import_hook): + """Executes a CGI script by importing it as a new module. + + This possibly reuses the module's main() function if it is defined and + takes no arguments. + + Basic technique lifted from PEP 338 and Python2.5's runpy module. See: + http://www.python.org/dev/peps/pep-0338/ + + See the section entitled "Import Statements and the Main Module" to understand + why a module named '__main__' cannot do relative imports. To get around this, + the requested module's path could be added to sys.path on each request. + + Args: + handler_path: CGI path stored in the application configuration (as a path + like 'foo/bar/baz.py'). Should not have $PYTHON_LIB references. + cgi_path: Absolute path to the CGI script file on disk. + import_hook: Instance of HardenedModulesHook to use for module loading. + + Returns: + True if the response code had an error status (e.g., 404), or False if it + did not. + + Raises: + Any kind of exception that could have been raised when loading the target + module, running a target script, or executing the application code itself. + """ + module_fullname, script_module, module_code = LoadTargetModule( + handler_path, cgi_path, import_hook) + script_module.__name__ = '__main__' + sys.modules['__main__'] = script_module + try: + if module_code: + exec module_code in script_module.__dict__ + else: + script_module.main() + + sys.stdout.flush() + sys.stdout.seek(0) + try: + headers = mimetools.Message(sys.stdout) + finally: + sys.stdout.seek(0, 2) + status_header = headers.get('status') + error_response = False + if status_header: + try: + status_code = int(status_header.split(' ', 1)[0]) + error_response = status_code >= 400 + except ValueError: + error_response = True + + if not error_response: + try: + parent_package = import_hook.GetParentPackage(module_fullname) + except Exception: + parent_package = None + + if parent_package is not None: + submodule = GetSubmoduleName(module_fullname) + setattr(parent_package, submodule, script_module) + + return error_response + finally: + script_module.__name__ = module_fullname + + +def ExecuteCGI(root_path, + handler_path, + cgi_path, + env, + infile, + outfile, + module_dict, + exec_script=ExecuteOrImportScript): + """Executes Python file in this process as if it were a CGI. + + Does not return an HTTP response line. CGIs should output headers followed by + the body content. + + The modules in sys.modules should be the same before and after the CGI is + executed, with the specific exception of encodings-related modules, which + cannot be reloaded and thus must always stay in sys.modules. + + Args: + root_path: Path to the root of the application. + handler_path: CGI path stored in the application configuration (as a path + like 'foo/bar/baz.py'). May contain $PYTHON_LIB references. + cgi_path: Absolute path to the CGI script file on disk. + env: Dictionary of environment variables to use for the execution. + infile: File-like object to read HTTP request input data from. + outfile: FIle-like object to write HTTP response data to. + module_dict: Dictionary in which application-loaded modules should be + preserved between requests. This removes the need to reload modules that + are reused between requests, significantly increasing load performance. + This dictionary must be separate from the sys.modules dictionary. + exec_script: Used for dependency injection. + """ + old_module_dict = sys.modules.copy() + old_builtin = __builtin__.__dict__.copy() + old_argv = sys.argv + old_stdin = sys.stdin + old_stdout = sys.stdout + old_env = os.environ.copy() + old_cwd = os.getcwd() + old_file_type = types.FileType + reset_modules = False + + try: + ClearAllButEncodingsModules(sys.modules) + sys.modules.update(module_dict) + sys.argv = [cgi_path] + sys.stdin = cStringIO.StringIO(infile.getvalue()) + sys.stdout = outfile + os.environ.clear() + os.environ.update(env) + before_path = sys.path[:] + cgi_dir = os.path.normpath(os.path.dirname(cgi_path)) + root_path = os.path.normpath(os.path.abspath(root_path)) + if cgi_dir.startswith(root_path + os.sep): + os.chdir(cgi_dir) + else: + os.chdir(root_path) + + hook = HardenedModulesHook(sys.modules) + sys.meta_path = [hook] + if hasattr(sys, 'path_importer_cache'): + sys.path_importer_cache.clear() + + __builtin__.file = FakeFile + __builtin__.open = FakeFile + types.FileType = FakeFile + + __builtin__.buffer = NotImplementedFakeClass + + logging.debug('Executing CGI with env:\n%s', pprint.pformat(env)) + try: + reset_modules = exec_script(handler_path, cgi_path, hook) + except SystemExit, e: + logging.debug('CGI exited with status: %s', e) + except: + reset_modules = True + raise + + finally: + sys.meta_path = [] + sys.path_importer_cache.clear() + + _ClearTemplateCache(sys.modules) + + module_dict.update(sys.modules) + ClearAllButEncodingsModules(sys.modules) + sys.modules.update(old_module_dict) + + __builtin__.__dict__.update(old_builtin) + sys.argv = old_argv + sys.stdin = old_stdin + sys.stdout = old_stdout + + sys.path[:] = before_path + + os.environ.clear() + os.environ.update(old_env) + os.chdir(old_cwd) + + types.FileType = old_file_type + + +class CGIDispatcher(URLDispatcher): + """Dispatcher that executes Python CGI scripts.""" + + def __init__(self, + module_dict, + root_path, + path_adjuster, + setup_env=SetupEnvironment, + exec_cgi=ExecuteCGI, + create_logging_handler=ApplicationLoggingHandler): + """Initializer. + + Args: + module_dict: Dictionary in which application-loaded modules should be + preserved between requests. This dictionary must be separate from the + sys.modules dictionary. + path_adjuster: Instance of PathAdjuster to use for finding absolute + paths of CGI files on disk. + setup_env, exec_cgi, create_logging_handler: Used for dependency + injection. + """ + self._module_dict = module_dict + self._root_path = root_path + self._path_adjuster = path_adjuster + self._setup_env = setup_env + self._exec_cgi = exec_cgi + self._create_logging_handler = create_logging_handler + + def Dispatch(self, + relative_url, + path, + headers, + infile, + outfile, + base_env_dict=None): + """Dispatches the Python CGI.""" + handler = self._create_logging_handler() + logging.getLogger().addHandler(handler) + before_level = logging.root.level + try: + env = {} + if base_env_dict: + env.update(base_env_dict) + cgi_path = self._path_adjuster.AdjustPath(path) + env.update(self._setup_env(cgi_path, relative_url, headers, infile)) + self._exec_cgi(self._root_path, + path, + cgi_path, + env, + infile, + outfile, + self._module_dict) + handler.AddDebuggingConsole(relative_url, env, outfile) + finally: + logging.root.level = before_level + logging.getLogger().removeHandler(handler) + + def __str__(self): + """Returns a string representation of this dispatcher.""" + return 'CGI dispatcher' + + +class LocalCGIDispatcher(CGIDispatcher): + """Dispatcher that executes local functions like they're CGIs. + + The contents of sys.modules will be preserved for local CGIs running this + dispatcher, but module hardening will still occur for any new imports. Thus, + be sure that any local CGIs have loaded all of their dependent modules + _before_ they are executed. + """ + + def __init__(self, module_dict, path_adjuster, cgi_func): + """Initializer. + + Args: + module_dict: Passed to CGIDispatcher. + path_adjuster: Passed to CGIDispatcher. + cgi_func: Callable function taking no parameters that should be + executed in a CGI environment in the current process. + """ + self._cgi_func = cgi_func + + def curried_exec_script(*args, **kwargs): + cgi_func() + return False + + def curried_exec_cgi(*args, **kwargs): + kwargs['exec_script'] = curried_exec_script + return ExecuteCGI(*args, **kwargs) + + CGIDispatcher.__init__(self, + module_dict, + '', + path_adjuster, + exec_cgi=curried_exec_cgi) + + def Dispatch(self, *args, **kwargs): + """Preserves sys.modules for CGIDispatcher.Dispatch.""" + self._module_dict.update(sys.modules) + CGIDispatcher.Dispatch(self, *args, **kwargs) + + def __str__(self): + """Returns a string representation of this dispatcher.""" + return 'Local CGI dispatcher for %s' % self._cgi_func + + + +class PathAdjuster(object): + """Adjusts application file paths to paths relative to the application or + external library directories.""" + + def __init__(self, root_path): + """Initializer. + + Args: + root_path: Path to the root of the application running on the server. + """ + self._root_path = os.path.abspath(root_path) + + def AdjustPath(self, path): + """Adjusts application file paths to relative to the application. + + More precisely this method adjusts application file path to paths + relative to the application or external library directories. + + Handler paths that start with $PYTHON_LIB will be converted to paths + relative to the google directory. + + Args: + path: File path that should be adjusted. + + Returns: + The adjusted path. + """ + if path.startswith(PYTHON_LIB_VAR): + path = os.path.join(os.path.dirname(os.path.dirname(google.__file__)), + path[len(PYTHON_LIB_VAR) + 1:]) + else: + path = os.path.join(self._root_path, path) + + return path + + + +class StaticFileConfigMatcher(object): + """Keeps track of file/directory specific application configuration. + + Specifically: + - Computes mime type based on URLMap and file extension. + - Decides on cache expiration time based on URLMap and default expiration. + + To determine the mime type, we first see if there is any mime-type property + on each URLMap entry. If non is specified, we use the mimetypes module to + guess the mime type from the file path extension, and use + application/octet-stream if we can't find the mimetype. + """ + + def __init__(self, + url_map_list, + path_adjuster, + default_expiration): + """Initializer. + + Args: + url_map_list: List of appinfo.URLMap objects. + If empty or None, then we always use the mime type chosen by the + mimetypes module. + path_adjuster: PathAdjuster object used to adjust application file paths. + default_expiration: String describing default expiration time for browser + based caching of static files. If set to None this disallows any + browser caching of static content. + """ + if default_expiration is not None: + self._default_expiration = appinfo.ParseExpiration(default_expiration) + else: + self._default_expiration = None + + self._patterns = [] + + if url_map_list: + for entry in url_map_list: + handler_type = entry.GetHandlerType() + if handler_type not in (appinfo.STATIC_FILES, appinfo.STATIC_DIR): + continue + + if handler_type == appinfo.STATIC_FILES: + regex = entry.upload + '$' + else: + path = entry.static_dir + if path[-1] == '/': + path = path[:-1] + regex = re.escape(path + os.path.sep) + r'(.*)' + + try: + path_re = re.compile(regex) + except re.error, e: + raise InvalidAppConfigError('regex %s does not compile: %s' % + (regex, e)) + + if self._default_expiration is None: + expiration = 0 + elif entry.expiration is None: + expiration = self._default_expiration + else: + expiration = appinfo.ParseExpiration(entry.expiration) + + self._patterns.append((path_re, entry.mime_type, expiration)) + + def IsStaticFile(self, path): + """Tests if the given path points to a "static" file. + + Args: + path: String containing the file's path relative to the app. + + Returns: + Boolean, True if the file was configured to be static. + """ + for (path_re, _, _) in self._patterns: + if path_re.match(path): + return True + return False + + def GetMimeType(self, path): + """Returns the mime type that we should use when serving the specified file. + + Args: + path: String containing the file's path relative to the app. + + Returns: + String containing the mime type to use. Will be 'application/octet-stream' + if we have no idea what it should be. + """ + for (path_re, mimetype, unused_expiration) in self._patterns: + if mimetype is not None: + the_match = path_re.match(path) + if the_match: + return mimetype + + unused_filename, extension = os.path.splitext(path) + return mimetypes.types_map.get(extension, 'application/octet-stream') + + def GetExpiration(self, path): + """Returns the cache expiration duration to be users for the given file. + + Args: + path: String containing the file's path relative to the app. + + Returns: + Integer number of seconds to be used for browser cache expiration time. + """ + for (path_re, unused_mimetype, expiration) in self._patterns: + the_match = path_re.match(path) + if the_match: + return expiration + + return self._default_expiration or 0 + + + + +def ReadDataFile(data_path, openfile=file): + """Reads a file on disk, returning a corresponding HTTP status and data. + + Args: + data_path: Path to the file on disk to read. + openfile: Used for dependency injection. + + Returns: + Tuple (status, data) where status is an HTTP response code, and data is + the data read; will be an empty string if an error occurred or the + file was empty. + """ + status = httplib.INTERNAL_SERVER_ERROR + data = "" + + try: + data_file = openfile(data_path, 'rb') + try: + data = data_file.read() + finally: + data_file.close() + status = httplib.OK + except (OSError, IOError), e: + logging.error('Error encountered reading file "%s":\n%s', data_path, e) + if e.errno in FILE_MISSING_EXCEPTIONS: + status = httplib.NOT_FOUND + else: + status = httplib.FORBIDDEN + + return status, data + + +class FileDispatcher(URLDispatcher): + """Dispatcher that reads data files from disk.""" + + def __init__(self, + path_adjuster, + static_file_config_matcher, + read_data_file=ReadDataFile): + """Initializer. + + Args: + path_adjuster: Instance of PathAdjuster to use for finding absolute + paths of data files on disk. + static_file_config_matcher: StaticFileConfigMatcher object. + read_data_file: Used for dependency injection. + """ + self._path_adjuster = path_adjuster + self._static_file_config_matcher = static_file_config_matcher + self._read_data_file = read_data_file + + def Dispatch(self, + relative_url, + path, + headers, + infile, + outfile, + base_env_dict=None): + """Reads the file and returns the response status and data.""" + full_path = self._path_adjuster.AdjustPath(path) + status, data = self._read_data_file(full_path) + content_type = self._static_file_config_matcher.GetMimeType(path) + expiration = self._static_file_config_matcher.GetExpiration(path) + + outfile.write('Status: %d\r\n' % status) + outfile.write('Content-type: %s\r\n' % content_type) + if expiration: + outfile.write('Expires: %s\r\n' + % email.Utils.formatdate(time.time() + expiration, + usegmt=True)) + outfile.write('Cache-Control: public, max-age=%i\r\n' % expiration) + outfile.write('\r\n') + outfile.write(data) + + def __str__(self): + """Returns a string representation of this dispatcher.""" + return 'File dispatcher' + + +_IGNORE_RESPONSE_HEADERS = frozenset([ + 'content-encoding', 'accept-encoding', 'transfer-encoding', + 'server', 'date', + ]) + + +def IgnoreHeadersRewriter(status_code, status_message, headers, body): + """Ignore specific response headers. + + Certain response headers cannot be modified by an Application. For a + complete list of these headers please see: + + http://code.google.com/appengine/docs/webapp/responseclass.html#Disallowed_HTTP_Response_Headers + + This rewriter simply removes those headers. + """ + for h in _IGNORE_RESPONSE_HEADERS: + if h in headers: + del headers[h] + + return status_code, status_message, headers, body + + +def ParseStatusRewriter(status_code, status_message, headers, body): + """Parse status header, if it exists. + + Handles the server-side 'status' header, which instructs the server to change + the HTTP response code accordingly. Handles the 'location' header, which + issues an HTTP 302 redirect to the client. Also corrects the 'content-length' + header to reflect actual content length in case extra information has been + appended to the response body. + + If the 'status' header supplied by the client is invalid, this method will + set the response to a 500 with an error message as content. + """ + location_value = headers.getheader('location') + status_value = headers.getheader('status') + if status_value: + response_status = status_value + del headers['status'] + elif location_value: + response_status = '%d Redirecting' % httplib.FOUND + else: + return status_code, status_message, headers, body + + status_parts = response_status.split(' ', 1) + status_code, status_message = (status_parts + [''])[:2] + try: + status_code = int(status_code) + except ValueError: + status_code = 500 + body = cStringIO.StringIO('Error: Invalid "status" header value returned.') + + return status_code, status_message, headers, body + + +def CacheRewriter(status_code, status_message, headers, body): + """Update the cache header.""" + if not 'Cache-Control' in headers: + headers['Cache-Control'] = 'no-cache' + if not 'Expires' in headers: + headers['Expires'] = 'Fri, 01 Jan 1990 00:00:00 GMT' + return status_code, status_message, headers, body + + +def ContentLengthRewriter(status_code, status_message, headers, body): + """Rewrite the Content-Length header. + + Even though Content-Length is not a user modifiable header, App Engine + sends a correct Content-Length to the user based on the actual response. + """ + current_position = body.tell() + body.seek(0, 2) + + headers['Content-Length'] = str(body.tell() - current_position) + body.seek(current_position) + return status_code, status_message, headers, body + + +def CreateResponseRewritersChain(): + """Create the default response rewriter chain. + + A response rewriter is the a function that gets a final chance to change part + of the dev_appservers response. A rewriter is not like a dispatcher in that + it is called after every request has been handled by the dispatchers + regardless of which dispatcher was used. + + The order in which rewriters are registered will be the order in which they + are used to rewrite the response. Modifications from earlier rewriters + are used as input to later rewriters. + + A response rewriter is a function that can rewrite the request in any way. + Thefunction can returned modified values or the original values it was + passed. + + A rewriter function has the following parameters and return values: + + Args: + status_code: Status code of response from dev_appserver or previous + rewriter. + status_message: Text corresponding to status code. + headers: mimetools.Message instance with parsed headers. NOTE: These + headers can contain its own 'status' field, but the default + dev_appserver implementation will remove this. Future rewriters + should avoid re-introducing the status field and return new codes + instead. + body: File object containing the body of the response. This position of + this file may not be at the start of the file. Any content before the + files position is considered not to be part of the final body. + + Returns: + status_code: Rewritten status code or original. + status_message: Rewritter message or original. + headers: Rewritten/modified headers or original. + body: Rewritten/modified body or original. + + Returns: + List of response rewriters. + """ + return [IgnoreHeadersRewriter, + ParseStatusRewriter, + CacheRewriter, + ContentLengthRewriter, + ] + + +def RewriteResponse(response_file, response_rewriters=None): + """Allows final rewrite of dev_appserver response. + + This function receives the unparsed HTTP response from the application + or internal handler, parses out the basic structure and feeds that structure + in to a chain of response rewriters. + + It also makes sure the final HTTP headers are properly terminated. + + For more about response rewriters, please see documentation for + CreateResponeRewritersChain. + + Args: + response_file: File-like object containing the full HTTP response including + the response code, all headers, and the request body. + response_rewriters: A list of response rewriters. If none is provided it + will create a new chain using CreateResponseRewritersChain. + + Returns: + Tuple (status_code, status_message, header, body) where: + status_code: Integer HTTP response status (e.g., 200, 302, 404, 500) + status_message: String containing an informational message about the + response code, possibly derived from the 'status' header, if supplied. + header: String containing the HTTP headers of the response, without + a trailing new-line (CRLF). + body: String containing the body of the response. + """ + if response_rewriters is None: + response_rewriters = CreateResponseRewritersChain() + + status_code = 200 + status_message = 'Good to go' + headers = mimetools.Message(response_file) + + for response_rewriter in response_rewriters: + status_code, status_message, headers, response_file = response_rewriter( + status_code, + status_message, + headers, + response_file) + + header_list = [] + for header in headers.headers: + header = header.rstrip('\n') + header = header.rstrip('\r') + header_list.append(header) + + header_data = '\r\n'.join(header_list) + '\r\n' + return status_code, status_message, header_data, response_file.read() + + + +class ModuleManager(object): + """Manages loaded modules in the runtime. + + Responsible for monitoring and reporting about file modification times. + Modules can be loaded from source or precompiled byte-code files. When a + file has source code, the ModuleManager monitors the modification time of + the source file even if the module itself is loaded from byte-code. + """ + + def __init__(self, modules): + """Initializer. + + Args: + modules: Dictionary containing monitored modules. + """ + self._modules = modules + self._default_modules = self._modules.copy() + self._save_path_hooks = sys.path_hooks[:] + self._modification_times = {} + + @staticmethod + def GetModuleFile(module, is_file=os.path.isfile): + """Helper method to try to determine modules source file. + + Args: + module: Module object to get file for. + is_file: Function used to determine if a given path is a file. + + Returns: + Path of the module's corresponding Python source file if it exists, or + just the module's compiled Python file. If the module has an invalid + __file__ attribute, None will be returned. + """ + module_file = getattr(module, '__file__', None) + if module_file is None: + return None + + source_file = module_file[:module_file.rfind('py') + 2] + + if is_file(source_file): + return source_file + return module.__file__ + + def AreModuleFilesModified(self): + """Determines if any monitored files have been modified. + + Returns: + True if one or more files have been modified, False otherwise. + """ + for name, (mtime, fname) in self._modification_times.iteritems(): + if name not in self._modules: + continue + + module = self._modules[name] + + if not os.path.isfile(fname): + return True + + if mtime != os.path.getmtime(fname): + return True + + return False + + def UpdateModuleFileModificationTimes(self): + """Records the current modification times of all monitored modules.""" + self._modification_times.clear() + for name, module in self._modules.items(): + if not isinstance(module, types.ModuleType): + continue + module_file = self.GetModuleFile(module) + if not module_file: + continue + try: + self._modification_times[name] = (os.path.getmtime(module_file), + module_file) + except OSError, e: + if e.errno not in FILE_MISSING_EXCEPTIONS: + raise e + + def ResetModules(self): + """Clear modules so that when request is run they are reloaded.""" + self._modules.clear() + self._modules.update(self._default_modules) + sys.path_hooks[:] = self._save_path_hooks + + + +def _ClearTemplateCache(module_dict=sys.modules): + """Clear template cache in webapp.template module. + + Attempts to load template module. Ignores failure. If module loads, the + template cache is cleared. + + Args: + module_dict: Used for dependency injection. + """ + template_module = module_dict.get('google.appengine.ext.webapp.template') + if template_module is not None: + template_module.template_cache.clear() + + + +def CreateRequestHandler(root_path, + login_url, + require_indexes=False, + static_caching=True): + """Creates a new BaseHTTPRequestHandler sub-class. + + This class will be used with the Python BaseHTTPServer module's HTTP server. + + Python's built-in HTTP server does not support passing context information + along to instances of its request handlers. This function gets around that + by creating a sub-class of the handler in a closure that has access to + this context information. + + Args: + root_path: Path to the root of the application running on the server. + login_url: Relative URL which should be used for handling user logins. + require_indexes: True if index.yaml is read-only gospel; default False. + static_caching: True if browser caching of static files should be allowed. + + Returns: + Sub-class of BaseHTTPRequestHandler. + """ + application_module_dict = SetupSharedModules(sys.modules) + + if require_indexes: + index_yaml_updater = None + else: + index_yaml_updater = dev_appserver_index.IndexYamlUpdater(root_path) + + application_config_cache = AppConfigCache() + + class DevAppServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """Dispatches URLs using patterns from a URLMatcher. + + The URLMatcher is created by loading an application's configuration file. + Executes CGI scripts in the local process so the scripts can use mock + versions of APIs. + + HTTP requests that correctly specify a user info cookie + (dev_appserver_login.COOKIE_NAME) will have the 'USER_EMAIL' environment + variable set accordingly. If the user is also an admin, the + 'USER_IS_ADMIN' variable will exist and be set to '1'. If the user is not + logged in, 'USER_EMAIL' will be set to the empty string. + + On each request, raises an InvalidAppConfigError exception if the + application configuration file in the directory specified by the root_path + argument is invalid. + """ + server_version = 'Development/1.0' + + module_dict = application_module_dict + module_manager = ModuleManager(application_module_dict) + + config_cache = application_config_cache + + rewriter_chain = CreateResponseRewritersChain() + + def __init__(self, *args, **kwargs): + """Initializer. + + Args: + args: Positional arguments passed to the superclass constructor. + kwargs: Keyword arguments passed to the superclass constructor. + """ + BaseHTTPServer.BaseHTTPRequestHandler.__init__(self, *args, **kwargs) + + def version_string(self): + """Returns server's version string used for Server HTTP header.""" + return self.server_version + + def do_GET(self): + """Handle GET requests.""" + self._HandleRequest() + + def do_POST(self): + """Handles POST requests.""" + self._HandleRequest() + + def do_PUT(self): + """Handle PUT requests.""" + self._HandleRequest() + + def do_HEAD(self): + """Handle HEAD requests.""" + self._HandleRequest() + + def do_OPTIONS(self): + """Handles OPTIONS requests.""" + self._HandleRequest() + + def do_DELETE(self): + """Handle DELETE requests.""" + self._HandleRequest() + + def do_TRACE(self): + """Handles TRACE requests.""" + self._HandleRequest() + + def _HandleRequest(self): + """Handles any type of request and prints exceptions if they occur.""" + server_name = self.headers.get('host') or self.server.server_name + server_name = server_name.split(':', 1)[0] + + env_dict = { + 'REQUEST_METHOD': self.command, + 'REMOTE_ADDR': self.client_address[0], + 'SERVER_SOFTWARE': self.server_version, + 'SERVER_NAME': server_name, + 'SERVER_PROTOCOL': self.protocol_version, + 'SERVER_PORT': str(self.server.server_port), + } + + full_url = GetFullURL(server_name, self.server.server_port, self.path) + if len(full_url) > MAX_URL_LENGTH: + msg = 'Requested URI too long: %s' % full_url + logging.error(msg) + self.send_response(httplib.REQUEST_URI_TOO_LONG, msg) + return + + tbhandler = cgitb.Hook(file=self.wfile).handle + try: + if self.module_manager.AreModuleFilesModified(): + self.module_manager.ResetModules() + + implicit_matcher = CreateImplicitMatcher(self.module_dict, + root_path, + login_url) + config, explicit_matcher = LoadAppConfig(root_path, self.module_dict, + cache=self.config_cache, + static_caching=static_caching) + if config.api_version != API_VERSION: + logging.error( + "API versions cannot be switched dynamically: %r != %r", + config.api_version, API_VERSION) + sys.exit(1) + env_dict['CURRENT_VERSION_ID'] = config.version + ".1" + env_dict['APPLICATION_ID'] = config.application + dispatcher = MatcherDispatcher(login_url, + [implicit_matcher, explicit_matcher]) + + if require_indexes: + dev_appserver_index.SetupIndexes(config.application, root_path) + + infile = cStringIO.StringIO() + infile.write(self.rfile.read( + int(self.headers.get('content-length', 0)))) + infile.seek(0) + + request_size = len(infile.getvalue()) + if request_size > MAX_REQUEST_SIZE: + msg = ('HTTP request was too large: %d. The limit is: %d.' + % (request_size, MAX_REQUEST_SIZE)) + logging.error(msg) + self.send_response(httplib.REQUEST_ENTITY_TOO_LARGE, msg) + return + + outfile = cStringIO.StringIO() + try: + dispatcher.Dispatch(self.path, + None, + self.headers, + infile, + outfile, + base_env_dict=env_dict) + finally: + self.module_manager.UpdateModuleFileModificationTimes() + + outfile.flush() + outfile.seek(0) + + status_code, status_message, header_data, body = ( + RewriteResponse(outfile, self.rewriter_chain)) + + runtime_response_size = len(outfile.getvalue()) + if runtime_response_size > MAX_RUNTIME_RESPONSE_SIZE: + status_code = 403 + status_message = 'Forbidden' + new_headers = [] + for header in header_data.split('\n'): + if not header.lower().startswith('content-length'): + new_headers.append(header) + header_data = '\n'.join(new_headers) + body = ('HTTP response was too large: %d. The limit is: %d.' + % (runtime_response_size, MAX_RUNTIME_RESPONSE_SIZE)) + + except yaml_errors.EventListenerError, e: + title = 'Fatal error when loading application configuration' + msg = '%s:\n%s' % (title, str(e)) + logging.error(msg) + self.send_response(httplib.INTERNAL_SERVER_ERROR, title) + self.wfile.write('Content-Type: text/html\n\n') + self.wfile.write('<pre>%s</pre>' % cgi.escape(msg)) + except: + msg = 'Exception encountered handling request' + logging.exception(msg) + self.send_response(httplib.INTERNAL_SERVER_ERROR, msg) + tbhandler() + else: + try: + self.send_response(status_code, status_message) + self.wfile.write(header_data) + self.wfile.write('\r\n') + if self.command != 'HEAD': + self.wfile.write(body) + elif body: + logging.warning('Dropping unexpected body in response ' + 'to HEAD request') + except (IOError, OSError), e: + if e.errno != errno.EPIPE: + raise e + except socket.error, e: + if len(e.args) >= 1 and e.args[0] != errno.EPIPE: + raise e + else: + if index_yaml_updater is not None: + index_yaml_updater.UpdateIndexYaml() + + def log_error(self, format, *args): + """Redirect error messages through the logging module.""" + logging.error(format, *args) + + def log_message(self, format, *args): + """Redirect log messages through the logging module.""" + logging.info(format, *args) + + return DevAppServerRequestHandler + + + +def ReadAppConfig(appinfo_path, parse_app_config=appinfo.LoadSingleAppInfo): + """Reads app.yaml file and returns its app id and list of URLMap instances. + + Args: + appinfo_path: String containing the path to the app.yaml file. + parse_app_config: Used for dependency injection. + + Returns: + AppInfoExternal instance. + + Raises: + If the config file could not be read or the config does not contain any + URLMap instances, this function will raise an InvalidAppConfigError + exception. + """ + try: + appinfo_file = file(appinfo_path, 'r') + except IOError, unused_e: + raise InvalidAppConfigError( + 'Application configuration could not be read from "%s"' % appinfo_path) + try: + return parse_app_config(appinfo_file) + finally: + appinfo_file.close() + + +def CreateURLMatcherFromMaps(root_path, + url_map_list, + module_dict, + default_expiration, + create_url_matcher=URLMatcher, + create_cgi_dispatcher=CGIDispatcher, + create_file_dispatcher=FileDispatcher, + create_path_adjuster=PathAdjuster, + normpath=os.path.normpath): + """Creates a URLMatcher instance from URLMap. + + Creates all of the correct URLDispatcher instances to handle the various + content types in the application configuration. + + Args: + root_path: Path to the root of the application running on the server. + url_map_list: List of appinfo.URLMap objects to initialize this + matcher with. Can be an empty list if you would like to add patterns + manually. + module_dict: Dictionary in which application-loaded modules should be + preserved between requests. This dictionary must be separate from the + sys.modules dictionary. + default_expiration: String describing default expiration time for browser + based caching of static files. If set to None this disallows any + browser caching of static content. + create_url_matcher: Used for dependency injection. + create_cgi_dispatcher: Used for dependency injection. + create_file_dispatcher: Used for dependency injection. + create_path_adjuster: Used for dependency injection. + normpath: Used for dependency injection. + + Returns: + Instance of URLMatcher with the supplied URLMap objects properly loaded. + + Raises: + InvalidAppConfigError: if the handler in url_map_list is an unknown type. + """ + url_matcher = create_url_matcher() + path_adjuster = create_path_adjuster(root_path) + cgi_dispatcher = create_cgi_dispatcher(module_dict, root_path, path_adjuster) + static_file_config_matcher = StaticFileConfigMatcher(url_map_list, + path_adjuster, + default_expiration) + file_dispatcher = create_file_dispatcher(path_adjuster, + static_file_config_matcher) + + FakeFile.SetStaticFileConfigMatcher(static_file_config_matcher) + + for url_map in url_map_list: + admin_only = url_map.login == appinfo.LOGIN_ADMIN + requires_login = url_map.login == appinfo.LOGIN_REQUIRED or admin_only + + handler_type = url_map.GetHandlerType() + if handler_type == appinfo.HANDLER_SCRIPT: + dispatcher = cgi_dispatcher + elif handler_type in (appinfo.STATIC_FILES, appinfo.STATIC_DIR): + dispatcher = file_dispatcher + else: + raise InvalidAppConfigError('Unknown handler type "%s"' % handler_type) + + regex = url_map.url + path = url_map.GetHandler() + if handler_type == appinfo.STATIC_DIR: + if regex[-1] == r'/': + regex = regex[:-1] + if path[-1] == os.path.sep: + path = path[:-1] + regex = '/'.join((re.escape(regex), '(.*)')) + if os.path.sep == '\\': + backref = r'\\1' + else: + backref = r'\1' + path = (normpath(path).replace('\\', '\\\\') + + os.path.sep + backref) + + url_matcher.AddURL(regex, + dispatcher, + path, + requires_login, admin_only) + + return url_matcher + + +class AppConfigCache(object): + """Cache used by LoadAppConfig. + + If given to LoadAppConfig instances of this class are used to cache contents + of the app config (app.yaml or app.yml) and the Matcher created from it. + + Code outside LoadAppConfig should treat instances of this class as opaque + objects and not access its members. + """ + + path = None + mtime = None + config = None + matcher = None + + +def LoadAppConfig(root_path, + module_dict, + cache=None, + static_caching=True, + read_app_config=ReadAppConfig, + create_matcher=CreateURLMatcherFromMaps): + """Creates a Matcher instance for an application configuration file. + + Raises an InvalidAppConfigError exception if there is anything wrong with + the application configuration file. + + Args: + root_path: Path to the root of the application to load. + module_dict: Dictionary in which application-loaded modules should be + preserved between requests. This dictionary must be separate from the + sys.modules dictionary. + cache: Instance of AppConfigCache or None. + static_caching: True if browser caching of static files should be allowed. + read_app_config: Used for dependency injection. + create_matcher: Used for dependency injection. + + Returns: + tuple: (AppInfoExternal, URLMatcher) + + Raises: + AppConfigNotFound: if an app.yaml file cannot be found. + """ + for appinfo_path in [os.path.join(root_path, 'app.yaml'), + os.path.join(root_path, 'app.yml')]: + + if os.path.isfile(appinfo_path): + if cache is not None: + mtime = os.path.getmtime(appinfo_path) + if cache.path == appinfo_path and cache.mtime == mtime: + return (cache.config, cache.matcher) + + cache.config = cache.matcher = cache.path = None + cache.mtime = mtime + + try: + config = read_app_config(appinfo_path, appinfo.LoadSingleAppInfo) + + if static_caching: + if config.default_expiration: + default_expiration = config.default_expiration + else: + default_expiration = '0' + else: + default_expiration = None + + matcher = create_matcher(root_path, + config.handlers, + module_dict, + default_expiration) + + FakeFile.SetSkippedFiles(config.skip_files) + + if cache is not None: + cache.path = appinfo_path + cache.config = config + cache.matcher = matcher + + return (config, matcher) + except gexcept.AbstractMethod: + pass + + raise AppConfigNotFoundError + + +def ReadCronConfig(croninfo_path, parse_cron_config=croninfo.LoadSingleCron): + """Reads cron.yaml file and returns a list of CronEntry instances. + + Args: + croninfo_path: String containing the path to the cron.yaml file. + parse_cron_config: Used for dependency injection. + + Returns: + A CronInfoExternal object. + + Raises: + If the config file is unreadable, empty or invalid, this function will + raise an InvalidAppConfigError or a MalformedCronConfiguration exception. + """ + try: + croninfo_file = file(croninfo_path, 'r') + except IOError, e: + raise InvalidAppConfigError( + 'Cron configuration could not be read from "%s": %s' + % (croninfo_path, e)) + try: + return parse_cron_config(croninfo_file) + finally: + croninfo_file.close() + + + +def SetupStubs(app_id, **config): + """Sets up testing stubs of APIs. + + Args: + app_id: Application ID being served. + config: keyword arguments. + + Keywords: + root_path: Root path to the directory of the application which should + contain the app.yaml, indexes.yaml, and queues.yaml files. + login_url: Relative URL which should be used for handling user login/logout. + datastore_path: Path to the file to store Datastore file stub data in. + history_path: Path to the file to store Datastore history in. + clear_datastore: If the datastore and history should be cleared on startup. + smtp_host: SMTP host used for sending test mail. + smtp_port: SMTP port. + smtp_user: SMTP user. + smtp_password: SMTP password. + enable_sendmail: Whether to use sendmail as an alternative to SMTP. + show_mail_body: Whether to log the body of emails. + remove: Used for dependency injection. + trusted: True if this app can access data belonging to other apps. This + behavior is different from the real app server and should be left False + except for advanced uses of dev_appserver. + """ + root_path = config.get('root_path', None) + login_url = config['login_url'] + datastore_path = config['datastore_path'] + history_path = config['history_path'] + clear_datastore = config['clear_datastore'] + require_indexes = config.get('require_indexes', False) + smtp_host = config.get('smtp_host', None) + smtp_port = config.get('smtp_port', 25) + smtp_user = config.get('smtp_user', '') + smtp_password = config.get('smtp_password', '') + enable_sendmail = config.get('enable_sendmail', False) + show_mail_body = config.get('show_mail_body', False) + remove = config.get('remove', os.remove) + trusted = config.get('trusted', False) + + os.environ['APPLICATION_ID'] = app_id + + if clear_datastore: + for path in (datastore_path, history_path): + if os.path.lexists(path): + logging.info('Attempting to remove file at %s', path) + try: + remove(path) + except OSError, e: + logging.warning('Removing file failed: %s', e) + + apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap() + + datastore = datastore_file_stub.DatastoreFileStub( + app_id, datastore_path, history_path, require_indexes=require_indexes, + trusted=trusted) + apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore) + + fixed_login_url = '%s?%s=%%s' % (login_url, + dev_appserver_login.CONTINUE_PARAM) + fixed_logout_url = '%s&%s' % (fixed_login_url, + dev_appserver_login.LOGOUT_PARAM) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'user', + user_service_stub.UserServiceStub(login_url=fixed_login_url, + logout_url=fixed_logout_url)) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'urlfetch', + urlfetch_stub.URLFetchServiceStub()) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'mail', + mail_stub.MailServiceStub(smtp_host, + smtp_port, + smtp_user, + smtp_password, + enable_sendmail=enable_sendmail, + show_mail_body=show_mail_body)) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'memcache', + memcache_stub.MemcacheServiceStub()) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'capability_service', + capability_stub.CapabilityServiceStub()) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'taskqueue', + taskqueue_stub.TaskQueueServiceStub(root_path=root_path)) + + apiproxy_stub_map.apiproxy.RegisterStub( + 'xmpp', + xmpp_service_stub.XmppServiceStub()) + + + + try: + from google.appengine.api.images import images_stub + apiproxy_stub_map.apiproxy.RegisterStub( + 'images', + images_stub.ImagesServiceStub()) + except ImportError, e: + logging.warning('Could not initialize images API; you are likely missing ' + 'the Python "PIL" module. ImportError: %s', e) + from google.appengine.api.images import images_not_implemented_stub + apiproxy_stub_map.apiproxy.RegisterStub( + 'images', + images_not_implemented_stub.ImagesNotImplementedServiceStub()) + + +def CreateImplicitMatcher(module_dict, + root_path, + login_url, + create_path_adjuster=PathAdjuster, + create_local_dispatcher=LocalCGIDispatcher, + create_cgi_dispatcher=CGIDispatcher): + """Creates a URLMatcher instance that handles internal URLs. + + Used to facilitate handling user login/logout, debugging, info about the + currently running app, etc. + + Args: + module_dict: Dictionary in the form used by sys.modules. + root_path: Path to the root of the application. + login_url: Relative URL which should be used for handling user login/logout. + create_path_adjuster: Used for dependedency injection. + create_local_dispatcher: Used for dependency injection. + create_cgi_dispatcher: Used for dependedency injection. + + Returns: + Instance of URLMatcher with appropriate dispatchers. + """ + url_matcher = URLMatcher() + path_adjuster = create_path_adjuster(root_path) + + login_dispatcher = create_local_dispatcher(sys.modules, path_adjuster, + dev_appserver_login.main) + url_matcher.AddURL(login_url, + login_dispatcher, + '', + False, + False) + + admin_dispatcher = create_cgi_dispatcher(module_dict, root_path, + path_adjuster) + url_matcher.AddURL('/_ah/admin(?:/.*)?', + admin_dispatcher, + DEVEL_CONSOLE_PATH, + False, + False) + + return url_matcher + + +def SetupTemplates(template_dir): + """Reads debugging console template files and initializes the console. + + Does nothing if templates have already been initialized. + + Args: + template_dir: Path to the directory containing the templates files. + + Raises: + OSError or IOError if any of the template files could not be read. + """ + if ApplicationLoggingHandler.AreTemplatesInitialized(): + return + + try: + header = open(os.path.join(template_dir, HEADER_TEMPLATE)).read() + script = open(os.path.join(template_dir, SCRIPT_TEMPLATE)).read() + middle = open(os.path.join(template_dir, MIDDLE_TEMPLATE)).read() + footer = open(os.path.join(template_dir, FOOTER_TEMPLATE)).read() + except (OSError, IOError): + logging.error('Could not read template files from %s', template_dir) + raise + + ApplicationLoggingHandler.InitializeTemplates(header, script, middle, footer) + + +def CreateServer(root_path, + login_url, + port, + template_dir, + serve_address='', + require_indexes=False, + allow_skipped_files=False, + static_caching=True, + python_path_list=sys.path, + sdk_dir=os.path.dirname(os.path.dirname(google.__file__))): + """Creates an new HTTPServer for an application. + + The sdk_dir argument must be specified for the directory storing all code for + the SDK so as to allow for the sandboxing of module access to work for any + and all SDK code. While typically this is where the 'google' package lives, + it can be in another location because of API version support. + + Args: + root_path: String containing the path to the root directory of the + application where the app.yaml file is. + login_url: Relative URL which should be used for handling user login/logout. + port: Port to start the application server on. + template_dir: Path to the directory in which the debug console templates + are stored. + serve_address: Address on which the server should serve. + require_indexes: True if index.yaml is read-only gospel; default False. + allow_skipped_files: True if skipped files should be accessible. + static_caching: True if browser caching of static files should be allowed. + python_path_list: Used for dependency injection. + sdk_dir: Directory where the SDK is stored. + + Returns: + Instance of BaseHTTPServer.HTTPServer that's ready to start accepting. + """ + absolute_root_path = os.path.realpath(root_path) + + SetupTemplates(template_dir) + FakeFile.SetAllowedPaths(absolute_root_path, + [sdk_dir, + template_dir]) + FakeFile.SetAllowSkippedFiles(allow_skipped_files) + + handler_class = CreateRequestHandler(absolute_root_path, + login_url, + require_indexes, + static_caching) + + if absolute_root_path not in python_path_list: + python_path_list.insert(0, absolute_root_path) + return HTTPServerWithScheduler((serve_address, port), handler_class) + + +class HTTPServerWithScheduler(BaseHTTPServer.HTTPServer): + """A BaseHTTPServer subclass that calls a method at a regular interval.""" + + def __init__(self, server_address, request_handler_class): + """Constructor. + + Args: + server_address: the bind address of the server. + request_handler_class: class used to handle requests. + """ + BaseHTTPServer.HTTPServer.__init__(self, server_address, + request_handler_class) + self._events = [] + + def get_request(self, time_func=time.time, select_func=select.select): + """Overrides the base get_request call. + + Args: + time_func: used for testing. + select_func: used for testing. + + Returns: + a (socket_object, address info) tuple. + """ + while True: + if self._events: + current_time = time_func() + next_eta = self._events[0][0] + delay = next_eta - current_time + else: + delay = DEFAULT_SELECT_DELAY + readable, _, _ = select_func([self.socket], [], [], max(delay, 0)) + if readable: + return self.socket.accept() + current_time = time_func() + if self._events and current_time >= self._events[0][0]: + unused_eta, runnable = heapq.heappop(self._events) + runnable() + + def AddEvent(self, eta, runnable): + """Add a runnable event to be run at the specified time. + + Args: + eta: when to run the event, in seconds since epoch. + runnable: a callable object. + """ + heapq.heappush(self._events, (eta, runnable)) |