From c96b89fdee098e13633686a1cff6d8f40d16a70a Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Tue, 19 May 2020 19:08:56 -0500 Subject: [PATCH 1/6] Improve module loading --- src/functions_framework/__init__.py | 20 ++++++++++++++++--- .../test_functions/module_is_correct/main.py | 5 ++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/functions_framework/__init__.py b/src/functions_framework/__init__.py index ce0e2fbf..641c715b 100644 --- a/src/functions_framework/__init__.py +++ b/src/functions_framework/__init__.py @@ -140,10 +140,24 @@ def create_app(target=None, source=None, signature_type=None): # Set the environment variable if it wasn't already os.environ["FUNCTION_SIGNATURE_TYPE"] = signature_type - # Load the source file - spec = importlib.util.spec_from_file_location("__main__", source) + # Load the source file: + # 1. Extract the module name from the source path + realpath = os.path.realpath(source) + directory, filename = os.path.split(realpath) + name, extension = os.path.splitext(filename) + + # 2. Create a new module + spec = importlib.util.spec_from_file_location(name, realpath) source_module = importlib.util.module_from_spec(spec) - sys.path.append(os.path.dirname(os.path.realpath(source))) + + # 3. Add the directory of the source to sys.path to allow the function to + # load modules relative to its location + sys.path.append(directory) + + # 4. Add the module to sys.modules + sys.modules[name] = source_module + + # 5. Execute the module spec.loader.exec_module(source_module) app = flask.Flask(target, template_folder=template_folder) diff --git a/tests/test_functions/module_is_correct/main.py b/tests/test_functions/module_is_correct/main.py index ed7a30cf..06d2f971 100644 --- a/tests/test_functions/module_is_correct/main.py +++ b/tests/test_functions/module_is_correct/main.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os.path import typing @@ -21,7 +22,9 @@ class TestClass: def function(request): # Ensure that the module for any object in this file is set correctly - assert TestClass.__mro__[0].__module__ == "__main__" + _, filename = os.path.split(__file__) + name, _ = os.path.splitext(filename) + assert TestClass.__mro__[0].__module__ == name # Ensure that calling `get_type_hints` on an object in this file succeeds assert typing.get_type_hints(TestClass) == {} From 7e254c9825db073ef6ce03c310deff43644f7316 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 20 May 2020 18:27:25 -0500 Subject: [PATCH 2/6] Remove --dry-run --- CHANGELOG.md | 2 ++ src/functions_framework/_cli.py | 10 ++-------- tests/test_cli.py | 12 ------------ 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 583e6655..bf8c7636 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Removed `--dry-run` flag ## [1.4.3] - 2020-05-14 ### Fixed diff --git a/src/functions_framework/_cli.py b/src/functions_framework/_cli.py index 4fe6e427..6f645f32 100644 --- a/src/functions_framework/_cli.py +++ b/src/functions_framework/_cli.py @@ -32,12 +32,6 @@ @click.option("--host", envvar="HOST", type=click.STRING, default="0.0.0.0") @click.option("--port", envvar="PORT", type=click.INT, default=8080) @click.option("--debug", envvar="DEBUG", is_flag=True) -@click.option("--dry-run", envvar="DRY_RUN", is_flag=True) -def _cli(target, source, signature_type, host, port, debug, dry_run): +def _cli(target, source, signature_type, host, port, debug): app = create_app(target, source, signature_type) - if dry_run: - click.echo("Function: {}".format(target)) - click.echo("URL: http://{}:{}/".format(host, port)) - click.echo("Dry run successful, shutting down.") - else: - create_server(app, debug).run(host, port) + create_server(app, debug).run(host, port) diff --git a/tests/test_cli.py b/tests/test_cli.py index aa4a901e..7613b649 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -69,18 +69,6 @@ def test_cli_no_arguments(): [pretend.call("foo", None, "event")], [pretend.call("0.0.0.0", 8080)], ), - ( - ["--target", "foo", "--dry-run"], - {}, - [pretend.call("foo", None, "http")], - [], - ), - ( - [], - {"FUNCTION_TARGET": "foo", "DRY_RUN": "True"}, - [pretend.call("foo", None, "http")], - [], - ), ( ["--target", "foo", "--host", "127.0.0.1"], {}, From 799e1b63a5e248b9f9ca05b24c283e7403c33afd Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 20 May 2020 18:29:13 -0500 Subject: [PATCH 3/6] Install and initialize Cloud Debugger --- CHANGELOG.md | 3 +++ setup.py | 1 + src/functions_framework/__init__.py | 8 +++++++ tests/test_debugger.py | 33 +++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+) create mode 100644 tests/test_debugger.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bf8c7636..c3010548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Enable the Google Cloud debugger + ### Changed - Removed `--dry-run` flag diff --git a/setup.py b/setup.py index 2a16fe7f..17bf312a 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ "click>=7.0,<8.0", "watchdog>=0.10.0", "gunicorn>=19.2.0,<21.0; platform_system!='Windows'", + "google-python-cloud-debugger>=1.0<3.0; platform_system=='Linux'", ], extras_require={"test": ["pytest", "tox"]}, entry_points={ diff --git a/src/functions_framework/__init__.py b/src/functions_framework/__init__.py index 641c715b..259c5fbc 100644 --- a/src/functions_framework/__init__.py +++ b/src/functions_framework/__init__.py @@ -140,6 +140,14 @@ def create_app(target=None, source=None, signature_type=None): # Set the environment variable if it wasn't already os.environ["FUNCTION_SIGNATURE_TYPE"] = signature_type + # Initialize the debugger if possible + try: + import googleclouddebugger + + googleclouddebugger.enable(module="[MODULE]", version="[VERSION]") + except ImportError: + pass + # Load the source file: # 1. Extract the module name from the source path realpath = os.path.realpath(source) diff --git a/tests/test_debugger.py b/tests/test_debugger.py new file mode 100644 index 00000000..d931cf81 --- /dev/null +++ b/tests/test_debugger.py @@ -0,0 +1,33 @@ +import pathlib +import sys + +import pretend + +from functions_framework import create_app + +TEST_FUNCTIONS_DIR = pathlib.Path.cwd() / "tests" / "test_functions" + + +def test_debugger_missing(monkeypatch): + monkeypatch.setitem(sys.modules, "googleclouddebugger", None) + + source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" + target = "function" + + create_app(target, source) + + +def test_debugger_present(monkeypatch): + googleclouddebugger = pretend.stub( + enable=pretend.call_recorder(lambda *a, **kw: None) + ) + monkeypatch.setitem(sys.modules, "googleclouddebugger", googleclouddebugger) + + source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" + target = "function" + + create_app(target, source) + + assert googleclouddebugger.enable.calls == [ + pretend.call(module="[MODULE]", version="[VERSION]"), + ] From 622655b252660d9d8ea9dc2f6d7c5b972b8b2e11 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 20 May 2020 18:29:22 -0500 Subject: [PATCH 4/6] Lazy-load the WSGI app The WSGI app must be created as late as possible for the debugger to correctly capture breakpoints --- src/functions_framework/__init__.py | 1 - src/functions_framework/_cli.py | 6 ++--- src/functions_framework/_http/__init__.py | 12 ++++----- src/functions_framework/_http/flask.py | 4 +-- src/functions_framework/_http/gunicorn.py | 10 +++++--- tests/test_cli.py | 30 +++++++++++------------ tests/test_http.py | 24 ++++++++++-------- 7 files changed, 46 insertions(+), 41 deletions(-) diff --git a/src/functions_framework/__init__.py b/src/functions_framework/__init__.py index 259c5fbc..b5d6f796 100644 --- a/src/functions_framework/__init__.py +++ b/src/functions_framework/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools import importlib.util import os.path import pathlib diff --git a/src/functions_framework/_cli.py b/src/functions_framework/_cli.py index 6f645f32..de15d098 100644 --- a/src/functions_framework/_cli.py +++ b/src/functions_framework/_cli.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os +from functools import partial import click @@ -33,5 +33,5 @@ @click.option("--port", envvar="PORT", type=click.INT, default=8080) @click.option("--debug", envvar="DEBUG", is_flag=True) def _cli(target, source, signature_type, host, port, debug): - app = create_app(target, source, signature_type) - create_server(app, debug).run(host, port) + load_app = partial(create_app, target, source, signature_type) + create_server(load_app, debug).run(host, port) diff --git a/src/functions_framework/_http/__init__.py b/src/functions_framework/_http/__init__.py index ca9b0f5c..d1a380b0 100644 --- a/src/functions_framework/_http/__init__.py +++ b/src/functions_framework/_http/__init__.py @@ -16,8 +16,8 @@ class HTTPServer: - def __init__(self, app, debug, **options): - self.app = app + def __init__(self, load_app, debug, **options): + self.load_app = load_app self.debug = debug self.options = options @@ -28,15 +28,15 @@ def __init__(self, app, debug, **options): from functions_framework._http.gunicorn import GunicornApplication self.server_class = GunicornApplication - except ImportError as e: + except ImportError: self.server_class = FlaskApplication def run(self, host, port): http_server = self.server_class( - self.app, host, port, self.debug, **self.options + self.load_app, host, port, self.debug, **self.options ) http_server.run() -def create_server(wsgi_app, debug, **options): - return HTTPServer(wsgi_app, debug, **options) +def create_server(load_app, debug, **options): + return HTTPServer(load_app, debug, **options) diff --git a/src/functions_framework/_http/flask.py b/src/functions_framework/_http/flask.py index b2edf563..f0414a54 100644 --- a/src/functions_framework/_http/flask.py +++ b/src/functions_framework/_http/flask.py @@ -14,8 +14,8 @@ class FlaskApplication: - def __init__(self, app, host, port, debug, **options): - self.app = app + def __init__(self, load_app, host, port, debug, **options): + self.app = load_app() self.host = host self.port = port self.debug = debug diff --git a/src/functions_framework/_http/gunicorn.py b/src/functions_framework/_http/gunicorn.py index 163b5317..67c217b5 100644 --- a/src/functions_framework/_http/gunicorn.py +++ b/src/functions_framework/_http/gunicorn.py @@ -15,8 +15,8 @@ import gunicorn.app.base -class GunicornApplication(gunicorn.app.base.BaseApplication): - def __init__(self, app, host, port, debug, **options): +class GunicornApplication(gunicorn.app.base.Application): + def __init__(self, load_app, host, port, debug, **options): self.options = { "bind": "%s:%s" % (host, port), "workers": 1, @@ -24,7 +24,7 @@ def __init__(self, app, host, port, debug, **options): "timeout": 0, } self.options.update(options) - self.app = app + self.load_app = load_app super().__init__() def load_config(self): @@ -32,4 +32,6 @@ def load_config(self): self.cfg.set(key, value) def load(self): - return self.app + # The WSGI app MUST be initalized here in order for the debugger to + # correctly trigger breakpoints + return self.load_app() diff --git a/tests/test_cli.py b/tests/test_cli.py index 7613b649..ab891a2b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,6 +19,7 @@ import functions_framework +from functions_framework import create_app from functions_framework._cli import _cli @@ -31,69 +32,68 @@ def test_cli_no_arguments(): @pytest.mark.parametrize( - "args, env, create_app_calls, run_calls", + "args, env, partial_calls, run_calls", [ ( ["--target", "foo"], {}, - [pretend.call("foo", None, "http")], + [pretend.call(create_app, "foo", None, "http")], [pretend.call("0.0.0.0", 8080)], ), ( [], {"FUNCTION_TARGET": "foo"}, - [pretend.call("foo", None, "http")], + [pretend.call(create_app, "foo", None, "http")], [pretend.call("0.0.0.0", 8080)], ), ( ["--target", "foo", "--source", "/path/to/source.py"], {}, - [pretend.call("foo", "/path/to/source.py", "http")], + [pretend.call(create_app, "foo", "/path/to/source.py", "http")], [pretend.call("0.0.0.0", 8080)], ), ( [], {"FUNCTION_TARGET": "foo", "FUNCTION_SOURCE": "/path/to/source.py"}, - [pretend.call("foo", "/path/to/source.py", "http")], + [pretend.call(create_app, "foo", "/path/to/source.py", "http")], [pretend.call("0.0.0.0", 8080)], ), ( ["--target", "foo", "--signature-type", "event"], {}, - [pretend.call("foo", None, "event")], + [pretend.call(create_app, "foo", None, "event")], [pretend.call("0.0.0.0", 8080)], ), ( [], {"FUNCTION_TARGET": "foo", "FUNCTION_SIGNATURE_TYPE": "event"}, - [pretend.call("foo", None, "event")], + [pretend.call(create_app, "foo", None, "event")], [pretend.call("0.0.0.0", 8080)], ), ( ["--target", "foo", "--host", "127.0.0.1"], {}, - [pretend.call("foo", None, "http")], + [pretend.call(create_app, "foo", None, "http")], [pretend.call("127.0.0.1", 8080)], ), ( ["--target", "foo", "--debug"], {}, - [pretend.call("foo", None, "http")], + [pretend.call(create_app, "foo", None, "http")], [pretend.call("0.0.0.0", 8080)], ), ( [], {"FUNCTION_TARGET": "foo", "DEBUG": "True"}, - [pretend.call("foo", None, "http")], + [pretend.call(create_app, "foo", None, "http")], [pretend.call("0.0.0.0", 8080)], ), ], ) -def test_cli(monkeypatch, args, env, create_app_calls, run_calls): +def test_cli(monkeypatch, args, env, partial_calls, run_calls): wsgi_server = pretend.stub(run=pretend.call_recorder(lambda *a, **kw: None)) - wsgi_app = pretend.stub(run=pretend.call_recorder(lambda *a, **kw: None)) - create_app = pretend.call_recorder(lambda *a, **kw: wsgi_app) - monkeypatch.setattr(functions_framework._cli, "create_app", create_app) + partial = pretend.call_recorder(lambda *a, **kw: pretend.stub()) + monkeypatch.setattr(functions_framework._cli, "partial", partial) create_server = pretend.call_recorder(lambda *a, **kw: wsgi_server) monkeypatch.setattr(functions_framework._cli, "create_server", create_server) @@ -101,5 +101,5 @@ def test_cli(monkeypatch, args, env, create_app_calls, run_calls): result = runner.invoke(_cli, args) assert result.exit_code == 0 - assert create_app.calls == create_app_calls + assert partial.calls == partial_calls assert wsgi_server.run.calls == run_calls diff --git a/tests/test_http.py b/tests/test_http.py index e9929d94..e1236ee3 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -26,12 +26,12 @@ def test_create_server(monkeypatch, debug): server_stub = pretend.stub() httpserver = pretend.call_recorder(lambda *a, **kw: server_stub) monkeypatch.setattr(functions_framework._http, "HTTPServer", httpserver) - wsgi_app = pretend.stub() + load_app = pretend.stub() options = {"a": pretend.stub(), "b": pretend.stub()} - functions_framework._http.create_server(wsgi_app, debug, **options) + functions_framework._http.create_server(load_app, debug, **options) - assert httpserver.calls == [pretend.call(wsgi_app, debug, **options)] + assert httpserver.calls == [pretend.call(load_app, debug, **options)] @pytest.mark.parametrize( @@ -44,7 +44,7 @@ def test_create_server(monkeypatch, debug): ], ) def test_httpserver(monkeypatch, debug, gunicorn_missing, expected): - app = pretend.stub() + load_app = pretend.stub() http_server = pretend.stub(run=pretend.call_recorder(lambda: None)) server_classes = { "flask": pretend.call_recorder(lambda *a, **kw: http_server), @@ -64,9 +64,9 @@ def test_httpserver(monkeypatch, debug, gunicorn_missing, expected): gunicorn, "GunicornApplication", server_classes["gunicorn"], ) - wrapper = functions_framework._http.HTTPServer(app, debug, **options) + wrapper = functions_framework._http.HTTPServer(load_app, debug, **options) - assert wrapper.app == app + assert wrapper.load_app == load_app assert wrapper.server_class == server_classes[expected] assert wrapper.options == options @@ -76,7 +76,7 @@ def test_httpserver(monkeypatch, debug, gunicorn_missing, expected): wrapper.run(host, port) assert wrapper.server_class.calls == [ - pretend.call(app, host, port, debug, **options) + pretend.call(load_app, host, port, debug, **options) ] assert http_server.run.calls == [pretend.call()] @@ -85,6 +85,7 @@ def test_httpserver(monkeypatch, debug, gunicorn_missing, expected): @pytest.mark.parametrize("debug", [True, False]) def test_gunicorn_application(debug): app = pretend.stub() + load_app = pretend.call_recorder(lambda: app) host = "1.2.3.4" port = "1234" options = {} @@ -92,10 +93,10 @@ def test_gunicorn_application(debug): import functions_framework._http.gunicorn gunicorn_app = functions_framework._http.gunicorn.GunicornApplication( - app, host, port, debug, **options + load_app, host, port, debug, **options ) - assert gunicorn_app.app == app + assert gunicorn_app.load_app == load_app assert gunicorn_app.options == { "bind": "%s:%s" % (host, port), "workers": 1, @@ -108,17 +109,19 @@ def test_gunicorn_application(debug): assert gunicorn_app.cfg.threads == 8 assert gunicorn_app.cfg.timeout == 0 assert gunicorn_app.load() == app + assert gunicorn_app.load_app.calls == [pretend.call()] @pytest.mark.parametrize("debug", [True, False]) def test_flask_application(debug): app = pretend.stub(run=pretend.call_recorder(lambda *a, **kw: None)) + load_app = pretend.call_recorder(lambda: app) host = pretend.stub() port = pretend.stub() options = {"a": pretend.stub(), "b": pretend.stub()} flask_app = functions_framework._http.flask.FlaskApplication( - app, host, port, debug, **options + load_app, host, port, debug, **options ) assert flask_app.app == app @@ -126,6 +129,7 @@ def test_flask_application(debug): assert flask_app.port == port assert flask_app.debug == debug assert flask_app.options == options + assert load_app.calls == [pretend.call()] flask_app.run() From b3c3241b4b69692d8861f924410d36ec1f61fa12 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 20 May 2020 22:40:44 -0500 Subject: [PATCH 5/6] Handle DefaultCredentialsError --- src/functions_framework/__init__.py | 3 +++ tests/test_debugger.py | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/functions_framework/__init__.py b/src/functions_framework/__init__.py index b5d6f796..a5fcd84a 100644 --- a/src/functions_framework/__init__.py +++ b/src/functions_framework/__init__.py @@ -142,10 +142,13 @@ def create_app(target=None, source=None, signature_type=None): # Initialize the debugger if possible try: import googleclouddebugger + from google.auth.exceptions import DefaultCredentialsError googleclouddebugger.enable(module="[MODULE]", version="[VERSION]") except ImportError: pass + except DefaultCredentialsError: + pass # Load the source file: # 1. Extract the module name from the source path diff --git a/tests/test_debugger.py b/tests/test_debugger.py index d931cf81..5cfcbcac 100644 --- a/tests/test_debugger.py +++ b/tests/test_debugger.py @@ -17,11 +17,35 @@ def test_debugger_missing(monkeypatch): create_app(target, source) -def test_debugger_present(monkeypatch): +def test_debugger_present_no_default_credentials(monkeypatch): + class DefaultCredentialsError(Exception): + pass + + googleauthexceptions = pretend.stub(DefaultCredentialsError=DefaultCredentialsError) + monkeypatch.setitem(sys.modules, "google.auth.exceptions", googleauthexceptions) + + googleclouddebugger = pretend.stub( + enable=pretend.call_recorder(pretend.raiser(DefaultCredentialsError)) + ) + monkeypatch.setitem(sys.modules, "googleclouddebugger", googleclouddebugger) + + source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" + target = "function" + + create_app(target, source) + + assert googleclouddebugger.enable.calls == [ + pretend.call(module="[MODULE]", version="[VERSION]"), + ] + + +def test_debugger_present_with_default_credentials(monkeypatch): googleclouddebugger = pretend.stub( enable=pretend.call_recorder(lambda *a, **kw: None) ) monkeypatch.setitem(sys.modules, "googleclouddebugger", googleclouddebugger) + googleauthexceptions = pretend.stub(DefaultCredentialsError=pretend.stub()) + monkeypatch.setitem(sys.modules, "google.auth.exceptions", googleauthexceptions) source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" target = "function" From 9c20b45c32fd1fd2cee2107e7e0b63dee78b4108 Mon Sep 17 00:00:00 2001 From: Dustin Ingram Date: Wed, 20 May 2020 22:57:23 -0500 Subject: [PATCH 6/6] Handle RuntimeError --- src/functions_framework/__init__.py | 2 ++ tests/test_debugger.py | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/functions_framework/__init__.py b/src/functions_framework/__init__.py index a5fcd84a..24602632 100644 --- a/src/functions_framework/__init__.py +++ b/src/functions_framework/__init__.py @@ -149,6 +149,8 @@ def create_app(target=None, source=None, signature_type=None): pass except DefaultCredentialsError: pass + except RuntimeError: + pass # Load the source file: # 1. Extract the module name from the source path diff --git a/tests/test_debugger.py b/tests/test_debugger.py index 5cfcbcac..6391e31d 100644 --- a/tests/test_debugger.py +++ b/tests/test_debugger.py @@ -39,11 +39,34 @@ class DefaultCredentialsError(Exception): ] +def test_debugger_present_and_already_attached(monkeypatch): + class DefaultCredentialsError(Exception): + pass + + googleauthexceptions = pretend.stub(DefaultCredentialsError=DefaultCredentialsError) + monkeypatch.setitem(sys.modules, "google.auth.exceptions", googleauthexceptions) + + googleclouddebugger = pretend.stub( + enable=pretend.call_recorder(pretend.raiser(RuntimeError)) + ) + monkeypatch.setitem(sys.modules, "googleclouddebugger", googleclouddebugger) + + source = TEST_FUNCTIONS_DIR / "http_trigger" / "main.py" + target = "function" + + create_app(target, source) + + assert googleclouddebugger.enable.calls == [ + pretend.call(module="[MODULE]", version="[VERSION]"), + ] + + def test_debugger_present_with_default_credentials(monkeypatch): googleclouddebugger = pretend.stub( enable=pretend.call_recorder(lambda *a, **kw: None) ) monkeypatch.setitem(sys.modules, "googleclouddebugger", googleclouddebugger) + googleauthexceptions = pretend.stub(DefaultCredentialsError=pretend.stub()) monkeypatch.setitem(sys.modules, "google.auth.exceptions", googleauthexceptions)