fastapi-gae-logging 0.0.12__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,287 @@
1
+ Metadata-Version: 2.3
2
+ Name: fastapi-gae-logging
3
+ Version: 0.2.0
4
+ Summary: Custom Cloud Logging handler for FastAPI (or any Starlette based) applications deployed in Google App Engine. Groups logs coming from the same request lifecycle and propagates the maximum log level throughout the request lifecycle using middleware and context management.
5
+ Author: Chris Karvouniaris
6
+ Author-email: Chris Karvouniaris <christos.karvouniaris247@gmail.com>
7
+ License: MIT
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: Operating System :: OS Independent
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Requires-Dist: google-cloud-logging>=3.13.0
12
+ Requires-Dist: starlette>=0.49.3
13
+ Requires-Python: >=3.9
14
+ Project-URL: Homepage, https://github.com/chrisK824/fastapi-gae-logging
15
+ Description-Content-Type: text/markdown
16
+
17
+ # fastapi-gae-logging
18
+ Custom Cloud Logging handler for FastAPI (or any Starlette based) applications deployed in Google App Engine to ease out logs analysis and monitoring through Google Cloud Log Explorer.
19
+
20
+ ## What problem does this package solve? Why do we need this?
21
+ When deploying FastAPI applications on Google App Engine, I encountered several challenges with logging and observability even when using the official package for this purpose, `google-cloud-logging`.
22
+
23
+ - **Scattered Logs**: Logs generated during a single request lifecycle were scattered across different log entries, making it difficult to trace the entire flow of a request, especially when troubleshooting issues.
24
+
25
+ - **Log Severity Mismatch**: The severity level of logs was not properly propagated throughout the request lifecycle. This meant that if an error occurred at any point in the request, the earlier logs did not reflect the severity of the final outcome, making it harder to identify problematic requests.
26
+
27
+ - **Payload Logging Issues**: Capturing and logging request payloads was cumbersome, requiring extra logging in the handlers and extra deployments. This led to incomplete logs, making it harder to reproduce issues or analyze request content.
28
+
29
+ - **Inconsistent Log Structures**: The default logging setup lacked a consistent structure, which made it challenging to filter, search, and analyze logs in the Google Cloud Log Explorer.
30
+
31
+ ## So what does it do?
32
+ The `fastapi-gae-logging` module addresses these problems by:
33
+
34
+ - **Grouping Logs by Request**: All logs generated during a request's lifecycle are grouped together, allowing for a complete view of the request flow in the Google Cloud Log Explorer. This makes it much easier to trace and troubleshoot issues.
35
+
36
+ - **Log Level Propagation**: The maximum log level observed during a request's lifecycle is propagated, ensuring that logs associated with a failed request reflect the appropriate severity. This improves the accuracy and utility of log searches based on severity.
37
+
38
+ - **Structured Payload Logging**: Request payloads are captured and logged in a structured format, even for non-dictionary JSON payloads. This ensures that all relevant request data is available for analysis, improving the ability to diagnose issues.
39
+
40
+
41
+ ## Install
42
+ `pip install fastapi-gae-logging`
43
+
44
+ ## Features:
45
+
46
+ - **Request Logs Grouping**: Groups logs from the same request lifecycle to simplify log analysis using Google Cloud Log Explorer. The logger name for grouping can be customized and defaults to the Google Cloud Project ID with '-request-logger' as a suffix.
47
+ - **Request Maximum Log Level Propagation**: Propagates the maximum log level throughout the request lifecycle, making it easier to search logs based on the severity of an issue.
48
+ - **Optional incoming request logging**: Opt in/out to log headers and payload of incoming requests into the `jsonPayload` field of the parent log.
49
+ - **Optional request headers logging**: Defaults to False. Headers dict lands into field `request_headers` in the `jsonPayload` of parent log.
50
+ - **Optional request Payload Logging**: Defaults to False. Incoming payload parsed lands into field `request_payload` in the `jsonPayload` of parent log. Parsing is based on content type with capability to override. Currenty embedded parsers for:
51
+ - `application/json`
52
+ - `application/x-www-form-urlencoded`
53
+ - `multipart/form-data`
54
+ - `text/plain`
55
+ - **Optional add-on log filters**:
56
+ - `GaeLogSizeLimitFilter` filter to drop log records if they exceed the maximum allowed size by google cloud logging.
57
+ - `GaeUrlib3FullPoolFilter` filter to drop noisy 'Connection pool is full' warning logs
58
+ from Google Cloud and App Engine internal libraries.
59
+
60
+ ## API
61
+
62
+ - Initialization
63
+
64
+ ```python
65
+ FastAPIGAELoggingHandler(
66
+ app: Starlette,
67
+ client: google.cloud.logging.Client,
68
+ request_logger_name: Optional[str] = None,
69
+ log_payload: bool = False,
70
+ log_headers: bool = False,
71
+ builtin_payload_parsers: Optional[List[PayloadParser.Defaults]] = None,
72
+ custom_payload_parsers: Optional[Dict[str, Callable[[Request], Awaitable[Any]]]] = None,
73
+ *args, **kwargs
74
+ )
75
+ ```
76
+
77
+ - Parameters
78
+ - **app** (FastAPI | Starlette): The FastAPI or Starlette application instance.
79
+ - **client** (google.cloud.logging.Client): The Google Cloud Logging Client instance. This is required to initialize the handler and retrieve the project ID.
80
+ - **request_logger_name** (Optional[str]): The name of the Cloud Logging logger to use for request logs. Defaults to the Google Cloud Project ID with the suffix '-request-logger'.
81
+ - **log_payload** (bool): Whether to log the request payload. If True, the payload for POST, PUT, PATCH, and DELETE requests will be captured and logged. Defaults to False.
82
+ - **log_headers** (bool): Whether to log the request headers. Defaults to False.
83
+ - **builtin_payload_parsers** (Optional[List[PayloadParser.Defaults]]): A list of built-in parser enums to enable (e.g., [PayloadParser.Defaults.JSON, PayloadParser.Defaults.FORM_URLENCODED]).
84
+ - **custom_payload_parsers** (Optional[Dict[str, Callable[[Request], Awaitable[Any]]]]): A dictionary mapping MIME types (e.g., 'application/custom+xml') to async parser coroutines. These coroutines must accept a Request object and return a serializable result to log. If provided, these will override default parsers for the specified content types.
85
+ - ***args**: Additional arguments to pass to the superclass constructor. Any argument you would pass to CloudLoggingHandler.
86
+ - ****kwargs**: Additional keyword arguments to pass to the superclass constructor. Any keyword argument you would pass to CloudLoggingHandler.
87
+
88
+
89
+ ## Example of usage
90
+
91
+ ```python
92
+ import logging
93
+ import os
94
+
95
+ from fastapi import FastAPI, File, Form, Request, UploadFile
96
+ from fastapi.exceptions import HTTPException
97
+ from fastapi.responses import JSONResponse
98
+
99
+ app = FastAPI()
100
+
101
+
102
+ async def custom_payload_parser_plain_text(request: Request):
103
+ # Custom parser for text/plain to demonstrate GAE handler extensibility.
104
+ # Needs to return a serializable value to be logged
105
+ body_bytes = await request.body()
106
+ incoming_payload = body_bytes.decode('utf-8')
107
+ return f"Parsed Plain Text: {incoming_payload}"
108
+
109
+
110
+ # Initialize GAE Logging
111
+ if os.getenv('GAE_ENV', '').startswith('standard'):
112
+ import google.cloud.logging
113
+ from google.cloud.logging_v2.handlers import setup_logging
114
+
115
+ from fastapi_gae_logging import (
116
+ FastAPIGAELoggingHandler,
117
+ GaeLogSizeLimitFilter,
118
+ GaeUrlib3FullPoolFilter,
119
+ PayloadParser,
120
+ )
121
+
122
+ client = google.cloud.logging.Client()
123
+ gae_log_handler = FastAPIGAELoggingHandler(
124
+ app=app,
125
+ client=client,
126
+ # Optional - opt in for logging payload and logs; defaults are False
127
+ log_headers=True,
128
+ log_payload=True,
129
+ # Optional - opt in for all built in payload parsers; applicable only if log_payload is set True
130
+ builtin_payload_parsers=[content_type for content_type in PayloadParser.Defaults],
131
+ # Optional - override built in payload parsers or provide more; applicable only if log_payload is set True
132
+ custom_payload_parsers={
133
+ "text/plain": custom_payload_parser_plain_text
134
+ }
135
+ )
136
+ setup_logging(handler=gae_log_handler)
137
+ # Optional - add extra filters for the logger
138
+ gae_log_handler.addFilter(GaeLogSizeLimitFilter())
139
+ gae_log_handler.addFilter(GaeUrlib3FullPoolFilter())
140
+
141
+
142
+ logging.getLogger().setLevel(logging.DEBUG)
143
+
144
+
145
+ @app.get("/info")
146
+ def info():
147
+ logging.debug("Step 1: Debugging diagnostic")
148
+ logging.info("Step 2: General information log")
149
+ return JSONResponse(
150
+ content={
151
+ "message": "info"
152
+ }
153
+ )
154
+
155
+
156
+ @app.get("/warning")
157
+ async def warning():
158
+ logging.debug("Step 1: Check system state")
159
+ logging.info("Step 2: State is normal")
160
+ logging.warning("Step 3: Resource usage approaching threshold")
161
+ return JSONResponse(
162
+ content={
163
+ "message": "warning"
164
+ }
165
+ )
166
+
167
+
168
+ @app.get("/error")
169
+ def error():
170
+ logging.debug("Step 1: Internal check")
171
+ logging.info("Step 2: Transaction started")
172
+ logging.warning("Step 3: Retry attempted")
173
+ logging.error("Step 4: Transaction failed after retries")
174
+ return JSONResponse(
175
+ content={
176
+ "message": "error"
177
+ }
178
+ )
179
+
180
+
181
+ @app.get("/exception")
182
+ def exception():
183
+ logging.debug("Step 1: Preparing logic")
184
+ logging.info("Step 2: Executing risky operation")
185
+ logging.error("Step 3: Critical failure detected")
186
+ raise ValueError("Simulated ValueError for GAE grouping demonstration")
187
+
188
+
189
+ @app.get("/http_exception")
190
+ def http_exception():
191
+ logging.debug("Step 1: Looking up resource")
192
+ logging.info("Step 2: Resource ID not found in database")
193
+ raise HTTPException(
194
+ status_code=404,
195
+ detail={
196
+ "error": "Resource not found"
197
+ }
198
+ )
199
+
200
+
201
+ @app.post("/post_payload")
202
+ async def post_payload(request: Request):
203
+ content_type = request.headers.get("content-type", "")
204
+ logging.debug(f"Handling POST request with Content-Type: {content_type}")
205
+
206
+ payload = None
207
+
208
+ # 1. Handle JSON
209
+ if "application/json" in content_type:
210
+ try:
211
+ payload = await request.json()
212
+ logging.info(f"Parsed as JSON: {payload}")
213
+ except Exception:
214
+ logging.warning("Failed to parse body as JSON")
215
+ payload = None
216
+
217
+ # 2. Handle Form URL-Encoded
218
+ elif "application/x-www-form-urlencoded" in content_type:
219
+ form_data = await request.form()
220
+ payload = dict(form_data)
221
+ logging.info(f"Parsed as Form URL-Encoded: {payload}")
222
+
223
+ # 3. Fallback for Plain Text or others
224
+ else:
225
+ body_bytes = await request.body()
226
+ payload = body_bytes.decode("utf-8", errors="replace")
227
+ logging.info(f"Parsed as Raw/Text: {payload}")
228
+
229
+ return JSONResponse(
230
+ content={
231
+ "mirror_response": payload,
232
+ "detected_type": str(type(payload)),
233
+ "content_type_received": content_type
234
+ },
235
+ status_code=200
236
+ )
237
+
238
+
239
+ @app.post("/post_form")
240
+ async def post_form(description: str = Form(...), file: UploadFile = File(...)): # noqa: B008
241
+ file_content = await file.read()
242
+
243
+ payload = {
244
+ "description": description,
245
+ "file_name": file.filename,
246
+ "content_type": file.content_type,
247
+ "file_size": len(file_content),
248
+ }
249
+
250
+ logging.info(f"Form submission processed: {payload}")
251
+ return JSONResponse(content={"mirror_response": payload}, status_code=200)
252
+
253
+ ```
254
+
255
+ ## How it looks in Google Cloud Log Explorer
256
+
257
+ ### Logger selection
258
+ ![alt text](https://github.com/chrisK824/fastapi-gae-logging/blob/main/images/logger_selection.jpg?raw=true)
259
+
260
+ ### Groupped logs with propagated log severity to the parent log
261
+
262
+ ![alt text](https://github.com/chrisK824/fastapi-gae-logging/blob/main/images/groupped_logs.jpg?raw=true)
263
+
264
+ ### Grouped logs in request with payload
265
+ ![alt text](https://github.com/chrisK824/fastapi-gae-logging/blob/main/images/request_with_payload.jpg?raw=true)
266
+
267
+ ## Dependencies
268
+ This tool is built upon the following packages:
269
+
270
+ - `starlette`: Starlette is a lightweight ASGI framework/toolkit, which is ideal for building async web services in Python. FastAPI is built on top of Starlette.
271
+ - `google-cloud-logging`: Google Cloud Logging API client library for logging and managing logs in Google Cloud Platform.
272
+
273
+
274
+ ## Implementation Concept
275
+ - **Middleware Integration**: A custom middleware integrates into FastAPI to intercept requests and log data after processing.The custom middleware is added to the FastAPI application during the initialization of the FastAPIGAELoggingHandler.
276
+ - **Context Management**: Uses context variables to manage request-specific data and metadata such as request payload, Google Cloud trace ID, start time of the incoming request and the maximum log level observed during the request lifecycle.
277
+ - **Log Interception**: A logging filter intercepts log records, injecting trace information and adjusting the maximum log level based on observed log severity.
278
+ - **Cloud Logging**: Utilizes Google Cloud Logging to group logs by request and propagate the maximum log level, enhancing observability and troubleshooting.
279
+ - **Structured Logging**: Parent log of the request-response lifecycle is structured and sent to Google Cloud Logging with additional context, such as the request method, URL, and user agent after the request has been processed and served.
280
+
281
+
282
+ ### Dev
283
+ - `uv sync --all-packages`
284
+ - Use `sample_app` folder for minimal Appengine app deployment of fastapi app that uses the local library src code via symlink.
285
+ - If symlink is broken for any reason, create it again from inside the `dev` folder: `ln -s ../src/fastapi_gae_logging/ .`
286
+ - Deploy the app: `gcloud app deploy --version=v1 default.yaml --project=<PROJECT_ID> --account <ACCOUNT_EMAIL>`
287
+ - Ping the sample app to generate logs for various cases in log explorer: `python3.12 ping_endpoints.py --project <PROJECT_ID>`
@@ -0,0 +1,271 @@
1
+ # fastapi-gae-logging
2
+ Custom Cloud Logging handler for FastAPI (or any Starlette based) applications deployed in Google App Engine to ease out logs analysis and monitoring through Google Cloud Log Explorer.
3
+
4
+ ## What problem does this package solve? Why do we need this?
5
+ When deploying FastAPI applications on Google App Engine, I encountered several challenges with logging and observability even when using the official package for this purpose, `google-cloud-logging`.
6
+
7
+ - **Scattered Logs**: Logs generated during a single request lifecycle were scattered across different log entries, making it difficult to trace the entire flow of a request, especially when troubleshooting issues.
8
+
9
+ - **Log Severity Mismatch**: The severity level of logs was not properly propagated throughout the request lifecycle. This meant that if an error occurred at any point in the request, the earlier logs did not reflect the severity of the final outcome, making it harder to identify problematic requests.
10
+
11
+ - **Payload Logging Issues**: Capturing and logging request payloads was cumbersome, requiring extra logging in the handlers and extra deployments. This led to incomplete logs, making it harder to reproduce issues or analyze request content.
12
+
13
+ - **Inconsistent Log Structures**: The default logging setup lacked a consistent structure, which made it challenging to filter, search, and analyze logs in the Google Cloud Log Explorer.
14
+
15
+ ## So what does it do?
16
+ The `fastapi-gae-logging` module addresses these problems by:
17
+
18
+ - **Grouping Logs by Request**: All logs generated during a request's lifecycle are grouped together, allowing for a complete view of the request flow in the Google Cloud Log Explorer. This makes it much easier to trace and troubleshoot issues.
19
+
20
+ - **Log Level Propagation**: The maximum log level observed during a request's lifecycle is propagated, ensuring that logs associated with a failed request reflect the appropriate severity. This improves the accuracy and utility of log searches based on severity.
21
+
22
+ - **Structured Payload Logging**: Request payloads are captured and logged in a structured format, even for non-dictionary JSON payloads. This ensures that all relevant request data is available for analysis, improving the ability to diagnose issues.
23
+
24
+
25
+ ## Install
26
+ `pip install fastapi-gae-logging`
27
+
28
+ ## Features:
29
+
30
+ - **Request Logs Grouping**: Groups logs from the same request lifecycle to simplify log analysis using Google Cloud Log Explorer. The logger name for grouping can be customized and defaults to the Google Cloud Project ID with '-request-logger' as a suffix.
31
+ - **Request Maximum Log Level Propagation**: Propagates the maximum log level throughout the request lifecycle, making it easier to search logs based on the severity of an issue.
32
+ - **Optional incoming request logging**: Opt in/out to log headers and payload of incoming requests into the `jsonPayload` field of the parent log.
33
+ - **Optional request headers logging**: Defaults to False. Headers dict lands into field `request_headers` in the `jsonPayload` of parent log.
34
+ - **Optional request Payload Logging**: Defaults to False. Incoming payload parsed lands into field `request_payload` in the `jsonPayload` of parent log. Parsing is based on content type with capability to override. Currenty embedded parsers for:
35
+ - `application/json`
36
+ - `application/x-www-form-urlencoded`
37
+ - `multipart/form-data`
38
+ - `text/plain`
39
+ - **Optional add-on log filters**:
40
+ - `GaeLogSizeLimitFilter` filter to drop log records if they exceed the maximum allowed size by google cloud logging.
41
+ - `GaeUrlib3FullPoolFilter` filter to drop noisy 'Connection pool is full' warning logs
42
+ from Google Cloud and App Engine internal libraries.
43
+
44
+ ## API
45
+
46
+ - Initialization
47
+
48
+ ```python
49
+ FastAPIGAELoggingHandler(
50
+ app: Starlette,
51
+ client: google.cloud.logging.Client,
52
+ request_logger_name: Optional[str] = None,
53
+ log_payload: bool = False,
54
+ log_headers: bool = False,
55
+ builtin_payload_parsers: Optional[List[PayloadParser.Defaults]] = None,
56
+ custom_payload_parsers: Optional[Dict[str, Callable[[Request], Awaitable[Any]]]] = None,
57
+ *args, **kwargs
58
+ )
59
+ ```
60
+
61
+ - Parameters
62
+ - **app** (FastAPI | Starlette): The FastAPI or Starlette application instance.
63
+ - **client** (google.cloud.logging.Client): The Google Cloud Logging Client instance. This is required to initialize the handler and retrieve the project ID.
64
+ - **request_logger_name** (Optional[str]): The name of the Cloud Logging logger to use for request logs. Defaults to the Google Cloud Project ID with the suffix '-request-logger'.
65
+ - **log_payload** (bool): Whether to log the request payload. If True, the payload for POST, PUT, PATCH, and DELETE requests will be captured and logged. Defaults to False.
66
+ - **log_headers** (bool): Whether to log the request headers. Defaults to False.
67
+ - **builtin_payload_parsers** (Optional[List[PayloadParser.Defaults]]): A list of built-in parser enums to enable (e.g., [PayloadParser.Defaults.JSON, PayloadParser.Defaults.FORM_URLENCODED]).
68
+ - **custom_payload_parsers** (Optional[Dict[str, Callable[[Request], Awaitable[Any]]]]): A dictionary mapping MIME types (e.g., 'application/custom+xml') to async parser coroutines. These coroutines must accept a Request object and return a serializable result to log. If provided, these will override default parsers for the specified content types.
69
+ - ***args**: Additional arguments to pass to the superclass constructor. Any argument you would pass to CloudLoggingHandler.
70
+ - ****kwargs**: Additional keyword arguments to pass to the superclass constructor. Any keyword argument you would pass to CloudLoggingHandler.
71
+
72
+
73
+ ## Example of usage
74
+
75
+ ```python
76
+ import logging
77
+ import os
78
+
79
+ from fastapi import FastAPI, File, Form, Request, UploadFile
80
+ from fastapi.exceptions import HTTPException
81
+ from fastapi.responses import JSONResponse
82
+
83
+ app = FastAPI()
84
+
85
+
86
+ async def custom_payload_parser_plain_text(request: Request):
87
+ # Custom parser for text/plain to demonstrate GAE handler extensibility.
88
+ # Needs to return a serializable value to be logged
89
+ body_bytes = await request.body()
90
+ incoming_payload = body_bytes.decode('utf-8')
91
+ return f"Parsed Plain Text: {incoming_payload}"
92
+
93
+
94
+ # Initialize GAE Logging
95
+ if os.getenv('GAE_ENV', '').startswith('standard'):
96
+ import google.cloud.logging
97
+ from google.cloud.logging_v2.handlers import setup_logging
98
+
99
+ from fastapi_gae_logging import (
100
+ FastAPIGAELoggingHandler,
101
+ GaeLogSizeLimitFilter,
102
+ GaeUrlib3FullPoolFilter,
103
+ PayloadParser,
104
+ )
105
+
106
+ client = google.cloud.logging.Client()
107
+ gae_log_handler = FastAPIGAELoggingHandler(
108
+ app=app,
109
+ client=client,
110
+ # Optional - opt in for logging payload and logs; defaults are False
111
+ log_headers=True,
112
+ log_payload=True,
113
+ # Optional - opt in for all built in payload parsers; applicable only if log_payload is set True
114
+ builtin_payload_parsers=[content_type for content_type in PayloadParser.Defaults],
115
+ # Optional - override built in payload parsers or provide more; applicable only if log_payload is set True
116
+ custom_payload_parsers={
117
+ "text/plain": custom_payload_parser_plain_text
118
+ }
119
+ )
120
+ setup_logging(handler=gae_log_handler)
121
+ # Optional - add extra filters for the logger
122
+ gae_log_handler.addFilter(GaeLogSizeLimitFilter())
123
+ gae_log_handler.addFilter(GaeUrlib3FullPoolFilter())
124
+
125
+
126
+ logging.getLogger().setLevel(logging.DEBUG)
127
+
128
+
129
+ @app.get("/info")
130
+ def info():
131
+ logging.debug("Step 1: Debugging diagnostic")
132
+ logging.info("Step 2: General information log")
133
+ return JSONResponse(
134
+ content={
135
+ "message": "info"
136
+ }
137
+ )
138
+
139
+
140
+ @app.get("/warning")
141
+ async def warning():
142
+ logging.debug("Step 1: Check system state")
143
+ logging.info("Step 2: State is normal")
144
+ logging.warning("Step 3: Resource usage approaching threshold")
145
+ return JSONResponse(
146
+ content={
147
+ "message": "warning"
148
+ }
149
+ )
150
+
151
+
152
+ @app.get("/error")
153
+ def error():
154
+ logging.debug("Step 1: Internal check")
155
+ logging.info("Step 2: Transaction started")
156
+ logging.warning("Step 3: Retry attempted")
157
+ logging.error("Step 4: Transaction failed after retries")
158
+ return JSONResponse(
159
+ content={
160
+ "message": "error"
161
+ }
162
+ )
163
+
164
+
165
+ @app.get("/exception")
166
+ def exception():
167
+ logging.debug("Step 1: Preparing logic")
168
+ logging.info("Step 2: Executing risky operation")
169
+ logging.error("Step 3: Critical failure detected")
170
+ raise ValueError("Simulated ValueError for GAE grouping demonstration")
171
+
172
+
173
+ @app.get("/http_exception")
174
+ def http_exception():
175
+ logging.debug("Step 1: Looking up resource")
176
+ logging.info("Step 2: Resource ID not found in database")
177
+ raise HTTPException(
178
+ status_code=404,
179
+ detail={
180
+ "error": "Resource not found"
181
+ }
182
+ )
183
+
184
+
185
+ @app.post("/post_payload")
186
+ async def post_payload(request: Request):
187
+ content_type = request.headers.get("content-type", "")
188
+ logging.debug(f"Handling POST request with Content-Type: {content_type}")
189
+
190
+ payload = None
191
+
192
+ # 1. Handle JSON
193
+ if "application/json" in content_type:
194
+ try:
195
+ payload = await request.json()
196
+ logging.info(f"Parsed as JSON: {payload}")
197
+ except Exception:
198
+ logging.warning("Failed to parse body as JSON")
199
+ payload = None
200
+
201
+ # 2. Handle Form URL-Encoded
202
+ elif "application/x-www-form-urlencoded" in content_type:
203
+ form_data = await request.form()
204
+ payload = dict(form_data)
205
+ logging.info(f"Parsed as Form URL-Encoded: {payload}")
206
+
207
+ # 3. Fallback for Plain Text or others
208
+ else:
209
+ body_bytes = await request.body()
210
+ payload = body_bytes.decode("utf-8", errors="replace")
211
+ logging.info(f"Parsed as Raw/Text: {payload}")
212
+
213
+ return JSONResponse(
214
+ content={
215
+ "mirror_response": payload,
216
+ "detected_type": str(type(payload)),
217
+ "content_type_received": content_type
218
+ },
219
+ status_code=200
220
+ )
221
+
222
+
223
+ @app.post("/post_form")
224
+ async def post_form(description: str = Form(...), file: UploadFile = File(...)): # noqa: B008
225
+ file_content = await file.read()
226
+
227
+ payload = {
228
+ "description": description,
229
+ "file_name": file.filename,
230
+ "content_type": file.content_type,
231
+ "file_size": len(file_content),
232
+ }
233
+
234
+ logging.info(f"Form submission processed: {payload}")
235
+ return JSONResponse(content={"mirror_response": payload}, status_code=200)
236
+
237
+ ```
238
+
239
+ ## How it looks in Google Cloud Log Explorer
240
+
241
+ ### Logger selection
242
+ ![alt text](https://github.com/chrisK824/fastapi-gae-logging/blob/main/images/logger_selection.jpg?raw=true)
243
+
244
+ ### Groupped logs with propagated log severity to the parent log
245
+
246
+ ![alt text](https://github.com/chrisK824/fastapi-gae-logging/blob/main/images/groupped_logs.jpg?raw=true)
247
+
248
+ ### Grouped logs in request with payload
249
+ ![alt text](https://github.com/chrisK824/fastapi-gae-logging/blob/main/images/request_with_payload.jpg?raw=true)
250
+
251
+ ## Dependencies
252
+ This tool is built upon the following packages:
253
+
254
+ - `starlette`: Starlette is a lightweight ASGI framework/toolkit, which is ideal for building async web services in Python. FastAPI is built on top of Starlette.
255
+ - `google-cloud-logging`: Google Cloud Logging API client library for logging and managing logs in Google Cloud Platform.
256
+
257
+
258
+ ## Implementation Concept
259
+ - **Middleware Integration**: A custom middleware integrates into FastAPI to intercept requests and log data after processing.The custom middleware is added to the FastAPI application during the initialization of the FastAPIGAELoggingHandler.
260
+ - **Context Management**: Uses context variables to manage request-specific data and metadata such as request payload, Google Cloud trace ID, start time of the incoming request and the maximum log level observed during the request lifecycle.
261
+ - **Log Interception**: A logging filter intercepts log records, injecting trace information and adjusting the maximum log level based on observed log severity.
262
+ - **Cloud Logging**: Utilizes Google Cloud Logging to group logs by request and propagate the maximum log level, enhancing observability and troubleshooting.
263
+ - **Structured Logging**: Parent log of the request-response lifecycle is structured and sent to Google Cloud Logging with additional context, such as the request method, URL, and user agent after the request has been processed and served.
264
+
265
+
266
+ ### Dev
267
+ - `uv sync --all-packages`
268
+ - Use `sample_app` folder for minimal Appengine app deployment of fastapi app that uses the local library src code via symlink.
269
+ - If symlink is broken for any reason, create it again from inside the `dev` folder: `ln -s ../src/fastapi_gae_logging/ .`
270
+ - Deploy the app: `gcloud app deploy --version=v1 default.yaml --project=<PROJECT_ID> --account <ACCOUNT_EMAIL>`
271
+ - Ping the sample app to generate logs for various cases in log explorer: `python3.12 ping_endpoints.py --project <PROJECT_ID>`
@@ -0,0 +1,58 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.9.17,<0.10.0"]
3
+ build-backend = "uv_build"
4
+
5
+ [tool.setuptools]
6
+ packages = ["fastapi_gae_logging"]
7
+
8
+ [tool.ruff]
9
+ line-length = 127
10
+ target-version = "py38"
11
+
12
+ [tool.ruff.lint]
13
+ select = [
14
+ "E",
15
+ "F",
16
+ "I",
17
+ "B",
18
+ "C90",
19
+ ]
20
+ ignore = []
21
+
22
+ [tool.ruff.lint.mccabe]
23
+ max-complexity = 20
24
+
25
+ [project]
26
+ name = "fastapi-gae-logging"
27
+ version = "0.2.0"
28
+ description = "Custom Cloud Logging handler for FastAPI (or any Starlette based) applications deployed in Google App Engine. Groups logs coming from the same request lifecycle and propagates the maximum log level throughout the request lifecycle using middleware and context management."
29
+ readme = "README.md"
30
+ requires-python = ">=3.9"
31
+ dependencies = [
32
+ "google-cloud-logging>=3.13.0",
33
+ "starlette>=0.49.3",
34
+ ]
35
+ classifiers = [
36
+ "Programming Language :: Python :: 3",
37
+ "Operating System :: OS Independent",
38
+ "License :: OSI Approved :: MIT License",
39
+ ]
40
+
41
+ [[project.authors]]
42
+ name = "Chris Karvouniaris"
43
+ email = "christos.karvouniaris247@gmail.com"
44
+
45
+ [project.license]
46
+ text = "MIT"
47
+
48
+ [project.urls]
49
+ Homepage = "https://github.com/chrisK824/fastapi-gae-logging"
50
+
51
+ [dependency-groups]
52
+ dev = [
53
+ "fastapi>=0.128.0",
54
+ "httpx>=0.28.1",
55
+ "python-multipart>=0.0.20",
56
+ "ruff>=0.14.10",
57
+ "uvicorn>=0.39.0",
58
+ ]
@@ -0,0 +1,50 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.9.17,<0.10.0"]
3
+ build-backend = "uv_build"
4
+
5
+ [tool.setuptools]
6
+ packages = ["fastapi_gae_logging"]
7
+
8
+ [project]
9
+ name = "fastapi-gae-logging"
10
+ version = "0.2.0"
11
+ description = "Custom Cloud Logging handler for FastAPI (or any Starlette based) applications deployed in Google App Engine. Groups logs coming from the same request lifecycle and propagates the maximum log level throughout the request lifecycle using middleware and context management."
12
+ readme = "README.md"
13
+ authors = [
14
+ { name = "Chris Karvouniaris", email = "christos.karvouniaris247@gmail.com" }
15
+ ]
16
+ license = { text = "MIT" }
17
+ requires-python = ">=3.9"
18
+ dependencies = [
19
+ "google-cloud-logging>=3.13.0",
20
+ "starlette>=0.49.3",
21
+ ]
22
+
23
+ classifiers = [
24
+ "Programming Language :: Python :: 3",
25
+ "Operating System :: OS Independent",
26
+ "License :: OSI Approved :: MIT License",
27
+ ]
28
+
29
+ [project.urls]
30
+ Homepage = "https://github.com/chrisK824/fastapi-gae-logging"
31
+
32
+ [dependency-groups]
33
+ dev = [
34
+ "fastapi>=0.128.0",
35
+ "httpx>=0.28.1",
36
+ "python-multipart>=0.0.20",
37
+ "ruff>=0.14.10",
38
+ "uvicorn>=0.39.0",
39
+ ]
40
+
41
+ [tool.ruff]
42
+ line-length = 127
43
+ target-version = "py38"
44
+
45
+ [tool.ruff.lint]
46
+ select = ["E", "F", "I", "B", "C90"]
47
+ ignore = []
48
+
49
+ [tool.ruff.lint.mccabe]
50
+ max-complexity = 20
@@ -0,0 +1,15 @@
1
+ from .fastapi_gae_logging import (
2
+ FastAPIGAELoggingHandler,
3
+ GaeLogSizeLimitFilter,
4
+ GaeUrlib3FullPoolFilter,
5
+ PayloadParser,
6
+ inject_gae_request_context,
7
+ )
8
+
9
+ __all__ = [
10
+ "FastAPIGAELoggingHandler",
11
+ "PayloadParser",
12
+ "GaeLogSizeLimitFilter",
13
+ "GaeUrlib3FullPoolFilter",
14
+ "inject_gae_request_context"
15
+ ]