singlestoredb 1.0.2__cp38-abi3-macosx_10_9_universal2.whl → 1.0.3__cp38-abi3-macosx_10_9_universal2.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of singlestoredb might be problematic. Click here for more details.

Binary file
singlestoredb/__init__.py CHANGED
@@ -13,7 +13,7 @@ Examples
13
13
 
14
14
  """
15
15
 
16
- __version__ = '1.0.2'
16
+ __version__ = '1.0.3'
17
17
 
18
18
  from typing import Any
19
19
 
singlestoredb/config.py CHANGED
@@ -253,6 +253,18 @@ register_option(
253
253
  environ=['SINGLESTOREDB_MANAGEMENT_TOKEN'],
254
254
  )
255
255
 
256
+ register_option(
257
+ 'management.base_url', 'string', check_str, 'https://api.singlestore.com',
258
+ 'Specifies the base URL for the management API.',
259
+ environ=['SINGLESTOREDB_MANAGEMENT_BASE_URL'],
260
+ )
261
+
262
+ register_option(
263
+ 'management.version', 'string', check_str, 'v1',
264
+ 'Specifies the version for the management API.',
265
+ environ=['SINGLESTOREDB_MANAGEMENT_VERSION'],
266
+ )
267
+
256
268
 
257
269
  #
258
270
  # Debugging options
@@ -22,18 +22,8 @@ def udf(
22
22
  func: Optional[Callable[..., Any]] = None,
23
23
  *,
24
24
  name: Optional[str] = None,
25
- database: Optional[str] = None,
26
- environment: Optional[str] = None,
27
- packages: Optional[Union[str, List[str]]] = None,
28
- resources: Optional[Union[str, List[str]]] = None,
29
- max_batch_size: int = 500,
30
- n_processes: int = 1,
31
- n_instances: int = 1,
32
25
  args: Optional[Union[DataType, List[DataType], Dict[str, DataType]]] = None,
33
26
  returns: Optional[str] = None,
34
- replace: bool = False,
35
- remote_service: Optional[str] = None,
36
- link: Optional[str] = None,
37
27
  data_format: Optional[str] = None,
38
28
  include_masks: bool = False,
39
29
  ) -> Callable[..., Any]:
@@ -46,27 +36,6 @@ def udf(
46
36
  The UDF to apply parameters to
47
37
  name : str, optional
48
38
  The name to use for the UDF in the database
49
- database : str, optional
50
- The database to create the functions in
51
- environment : str, optional
52
- The environment to create for the functions
53
- packages : str or list-of-strs, optional
54
- The package dependency specifications. Strings must be in the
55
- format used in requirements.txt files.
56
- max_match_size : int, optional
57
- The number of rows to batch in the server before sending
58
- them to the UDF application
59
- n_processes : int, optional
60
- The number of sub-processes to spin up to process sub-batches
61
- of rows of data. This may be used if the UDF is CPU-intensive
62
- and there are free CPUs in the server the web application
63
- is running in. If the UDF is very short-running, setting this
64
- parameter to greater than one will likey cause the UDF to
65
- run slower since the overhead of the extra processes and moving
66
- data would be the limiting factor.
67
- n_instances : int, optional
68
- The number of runtime environments to use to handle data
69
- processing requests
70
39
  args : str | Callable | List[str | Callable] | Dict[str, str | Callable], optional
71
40
  Specifies the data types of the function arguments. Typically,
72
41
  the function data types are derived from the function parameter
@@ -83,14 +52,6 @@ def udf(
83
52
  returns : str, optional
84
53
  Specifies the return data type of the function. If not specified,
85
54
  the type annotation from the function is used.
86
- replace : bool, optional
87
- Should an existing function of the same name be replaced when
88
- creating the function in the database?
89
- remote_service : str, optional
90
- URL of the remote service that handles this function. If using an
91
- environment, this URL is generated automatically.
92
- link : str, optional
93
- Name of link to use to connect to remote service
94
55
  data_format : str, optional
95
56
  The data format of each parameter: python, pandas, arrow, polars
96
57
  include_masks : bool, optional
@@ -103,10 +64,6 @@ def udf(
103
64
  Callable
104
65
 
105
66
  """
106
- assert max_batch_size >= 1
107
- assert n_processes >= 1
108
- assert n_instances >= 1
109
-
110
67
  if args is None:
111
68
  pass
112
69
  elif isinstance(args, (list, tuple)):
@@ -153,18 +110,8 @@ def udf(
153
110
  _singlestoredb_attrs = { # type: ignore
154
111
  k: v for k, v in dict(
155
112
  name=name,
156
- database=database,
157
- environment=environment,
158
- packages=listify(packages),
159
- resources=listify(resources),
160
- max_batch_size=max(1, int(max_batch_size)),
161
- n_processes=max(1, int(n_processes)),
162
- n_instances=max(1, int(n_instances)),
163
113
  args=args,
164
114
  returns=returns,
165
- replace=bool(replace),
166
- remote_service=remote_service,
167
- link=link,
168
115
  data_format=data_format,
169
116
  include_masks=include_masks,
170
117
  ).items() if v is not None
@@ -18,15 +18,17 @@ An example of starting a server is shown below.
18
18
 
19
19
  Example
20
20
  -------
21
- $ SINGLESTOREDB_EXT_FUNCTIONS='myfuncs.[percentile_90,percentile_95]' \
21
+ $ SINGLESTOREDB_EXT_FUNCTIONS='myfuncs.[percentage_90,percentage_95]' \
22
22
  uvicorn --factory singlestoredb.functions.ext:create_app
23
23
 
24
24
  '''
25
25
  import importlib.util
26
26
  import io
27
27
  import itertools
28
+ import json
28
29
  import os
29
- import urllib
30
+ import re
31
+ import secrets
30
32
  from types import ModuleType
31
33
  from typing import Any
32
34
  from typing import Awaitable
@@ -36,6 +38,7 @@ from typing import Iterable
36
38
  from typing import List
37
39
  from typing import Optional
38
40
  from typing import Sequence
41
+ from typing import Set
39
42
  from typing import Tuple
40
43
  from typing import Union
41
44
 
@@ -196,8 +199,12 @@ def create_app( # noqa: C901
196
199
  Iterable[ModuleType],
197
200
  ]
198
201
  ] = None,
199
-
200
-
202
+ app_mode: str = 'remote',
203
+ url: str = 'http://localhost:8000/invoke',
204
+ data_format: str = 'rowdat_1',
205
+ data_version: str = '1.0',
206
+ link_config: Optional[Dict[str, Any]] = None,
207
+ link_credentials: Optional[Dict[str, Any]] = None,
201
208
  ) -> Callable[..., Any]:
202
209
  '''
203
210
  Create an external function application.
@@ -216,6 +223,20 @@ def create_app( # noqa: C901
216
223
  * Multiple functions : <pkg1>.[<func1-name,func2-name,...]
217
224
  * Function aliases : <pkg1>.[<func1@alias1,func2@alias2,...]
218
225
  * Multiple packages : <pkg1>.<func1>:<pkg2>.<func2>
226
+ app_mode : str, optional
227
+ The mode of operation for the application: remote or collocated
228
+ url : str, optional
229
+ The URL of the function API
230
+ data_format : str, optional
231
+ The format of the data rows: 'rowdat_1' or 'json'
232
+ data_version : str, optional
233
+ The version of the call format to expect: '1.0'
234
+ link_config : Dict[str, Any], optional
235
+ The CONFIG section of a LINK definition. This dictionary gets
236
+ converted to JSON for the CREATE LINK call.
237
+ link_credentials : Dict[str, Any], optional
238
+ The CREDENTIALS section of a LINK definition. This dictionary gets
239
+ converted to JSON for the CREATE LINK call.
219
240
 
220
241
  Returns
221
242
  -------
@@ -469,7 +490,7 @@ def create_app( # noqa: C901
469
490
  # Handle api reflection
470
491
  elif method == 'GET' and path == show_create_function_path:
471
492
  host = headers.get(b'host', b'localhost:80')
472
- url = f'{scope["scheme"]}://{host.decode("utf-8")}/invoke'
493
+ reflected_url = f'{scope["scheme"]}://{host.decode("utf-8")}/invoke'
473
494
  data_format = 'json' if b'json' in content_type else 'rowdat_1'
474
495
 
475
496
  syntax = []
@@ -478,7 +499,7 @@ def create_app( # noqa: C901
478
499
  syntax.append(
479
500
  signature_to_sql(
480
501
  endpoint._ext_func_signature, # type: ignore
481
- base_url=url,
502
+ url=url or reflected_url,
482
503
  data_format=data_format,
483
504
  ),
484
505
  )
@@ -496,35 +517,88 @@ def create_app( # noqa: C901
496
517
  out['body'] = body
497
518
  await send(out)
498
519
 
520
+ def _create_link(
521
+ config: Optional[Dict[str, Any]],
522
+ credentials: Optional[Dict[str, Any]],
523
+ ) -> Tuple[str, str]:
524
+ """Generate CREATE LINK command."""
525
+ if not config and not credentials:
526
+ return '', ''
527
+
528
+ link_name = f'link_{secrets.token_hex(16)}'
529
+ out = [f'CREATE LINK {link_name} AS HTTP']
530
+
531
+ if config:
532
+ out.append(f"CONFIG '{json.dumps(config)}'")
533
+
534
+ if credentials:
535
+ out.append(f"CREDENTIALS '{json.dumps(credentials)}'")
536
+
537
+ return link_name, ' '.join(out) + ';'
538
+
539
+ def _locate_app_functions(cur: Any) -> Tuple[Set[str], Set[str]]:
540
+ """Locate all current functions and links belonging to this app."""
541
+ funcs, links = set(), set()
542
+ cur.execute('SHOW FUNCTIONS')
543
+ for name, ftype, _, _, _, link in list(cur):
544
+ # Only look at external functions
545
+ if 'external' not in ftype.lower():
546
+ continue
547
+ # See if function URL matches url
548
+ cur.execute(f'SHOW CREATE FUNCTION `{name}`')
549
+ for fname, _, code, *_ in list(cur):
550
+ m = re.search(r" (?:\w+) SERVICE '([^']+)'", code)
551
+ if m and m.group(1) == url:
552
+ funcs.add(fname)
553
+ if link:
554
+ links.add(link)
555
+ return funcs, links
556
+
499
557
  def show_create_functions(
500
- base_url: str = 'http://localhost:8000',
501
- data_format: str = 'rowdat_1',
558
+ replace: bool = False,
502
559
  ) -> List[str]:
560
+ """Generate CREATE FUNCTION calls."""
561
+ if not endpoints:
562
+ return []
563
+
503
564
  out = []
565
+ link = ''
566
+ if app_mode.lower() == 'remote':
567
+ link, link_str = _create_link(link_config, link_credentials)
568
+ if link and link_str:
569
+ out.append(link_str)
570
+
504
571
  for key, endpoint in endpoints.items():
505
572
  out.append(
506
573
  signature_to_sql(
507
574
  endpoint._ext_func_signature, # type: ignore
508
- base_url=urllib.parse.urljoin(base_url, '/invoke'),
575
+ url=url,
509
576
  data_format=data_format,
577
+ app_mode=app_mode,
578
+ replace=replace,
579
+ link=link or None,
510
580
  ),
511
581
  )
582
+
512
583
  return out
513
584
 
514
585
  app.show_create_functions = show_create_functions # type: ignore
515
586
 
516
587
  def register_functions(
517
588
  *connection_args: Any,
518
- base_url: str = 'http://localhost:8000',
519
- data_format: str = 'rowdat_1',
589
+ replace: bool = False,
520
590
  **connection_kwargs: Any,
521
591
  ) -> None:
592
+ """Register functions with the database."""
522
593
  with connection.connect(*connection_args, **connection_kwargs) as conn:
523
594
  with conn.cursor() as cur:
524
- for func in app.show_create_functions( # type: ignore
525
- base_url=base_url,
526
- data_format=data_format,
527
- ): # type: ignore
595
+ if replace:
596
+ funcs, links = _locate_app_functions(cur)
597
+ for fname in funcs:
598
+ cur.execute(f'DROP FUNCTION IF EXISTS `{fname}`')
599
+ for link in links:
600
+ cur.execute(f'DROP LINK {link}')
601
+ for func in app.show_create_functions(replace=replace): # type: ignore
528
602
  cur.execute(func)
529
603
 
530
604
  app.register_functions = register_functions # type: ignore
@@ -533,10 +607,14 @@ def create_app( # noqa: C901
533
607
  *connection_args: Any,
534
608
  **connection_kwargs: Any,
535
609
  ) -> None:
610
+ """Drop registered functions from database."""
536
611
  with connection.connect(*connection_args, **connection_kwargs) as conn:
537
612
  with conn.cursor() as cur:
538
- for key in endpoints.keys():
539
- cur.execute(f'DROP FUNCTION IF EXISTS `{key.decode("utf8")}`')
613
+ funcs, links = _locate_app_functions(cur)
614
+ for fname in funcs:
615
+ cur.execute(f'DROP FUNCTION IF EXISTS `{fname}`')
616
+ for link in links:
617
+ cur.execute(f'DROP LINK {link}')
540
618
 
541
619
  app.drop_functions = drop_functions # type: ignore
542
620
 
@@ -544,8 +622,8 @@ def create_app( # noqa: C901
544
622
  name: str,
545
623
  data_in: io.BytesIO,
546
624
  data_out: io.BytesIO,
547
- data_format: str = 'rowdat_1',
548
- data_version: str = '1.0',
625
+ data_format: str = data_format,
626
+ data_version: str = data_version,
549
627
  ) -> None:
550
628
 
551
629
  async def receive() -> Dict[str, Any]:
@@ -0,0 +1,306 @@
1
+ #!/usr/bin/env python
2
+ '''
3
+ Module for creating collocated Python UDFs
4
+
5
+ This module implements the collocated form of external functions for
6
+ SingleStoreDB. This mode uses a socket for control communications
7
+ and memory mapped files for passing the data to and from the UDF.
8
+
9
+ The command below is a sample invocation. It exports all functions
10
+ within the `myfuncs` Python module that have a `@udf` decorator on
11
+ them. The `--db` option specifies a database connection string.
12
+ If this exists, the UDF application will connect to the database
13
+ and register all functions. The `--replace-existing` option indicates
14
+ that existing functions should be replaced::
15
+
16
+ python -m singlestoredb.functions.ext.mmap \
17
+ --db=root:@127.0.0.1:9306/cosmeticshop --replace-existing \
18
+ myfuncs
19
+
20
+ The `myfuncs` package can be any Python package in your Python path.
21
+ It must contain functions marked with a `@udf` decorator and the
22
+ types must be annotated or specified using the `@udf` decorator
23
+ similar to the following::
24
+
25
+ from singlestoredb.functions import udf
26
+
27
+ @udf
28
+ def print_it(x2: float, x3: str) -> str:
29
+ return int(x2) * x3
30
+
31
+ @udf.pandas
32
+ def print_it_pandas(x2: float, x3: str) -> str:
33
+ return x2.astype(np.int64) * x3.astype(str)
34
+
35
+ With the functions registered, you can now run the UDFs::
36
+
37
+ SELECT print_it(3.14, 'my string');
38
+ SELECT print_it_pandas(3.14, 'my string');
39
+
40
+ '''
41
+ import argparse
42
+ import array
43
+ import asyncio
44
+ import io
45
+ import logging
46
+ import mmap
47
+ import multiprocessing
48
+ import os
49
+ import secrets
50
+ import socket
51
+ import struct
52
+ import sys
53
+ import tempfile
54
+ import threading
55
+ import traceback
56
+ from typing import Any
57
+
58
+ from . import asgi
59
+
60
+
61
+ logger = logging.getLogger('singlestoredb.functions.ext.mmap')
62
+ handler = logging.StreamHandler()
63
+ formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
64
+ handler.setFormatter(formatter)
65
+ logger.addHandler(handler)
66
+ logger.setLevel(logging.INFO)
67
+
68
+
69
+ def _handle_request(app: Any, connection: Any, client_address: Any) -> None:
70
+ '''
71
+ Handle function call request.
72
+
73
+ Parameters:
74
+ app : ASGI app
75
+ An ASGI application from the singlestoredb.functions.ext.asgi module
76
+ connection : socket connection
77
+ Socket connection for function control messages
78
+ client_address : string
79
+ Address of connecting client
80
+
81
+ '''
82
+ logger.info('connection from {}'.format(str(connection).split(', ')[0][-4:]))
83
+
84
+ # Receive the request header. Format:
85
+ # server version: uint64
86
+ # length of function name: uint64
87
+ buf = connection.recv(16)
88
+ version, namelen = struct.unpack('<qq', buf)
89
+
90
+ # Python's recvmsg returns a tuple. We only really care about the first
91
+ # two parts. The recvmsg call has a weird way of specifying the size for
92
+ # the file descriptor array; basically, we're indicating we want to read
93
+ # two 32-bit ints (for the input and output files).
94
+ fd_model = array.array('i', [0, 0])
95
+ msg, ancdata, flags, addr = connection.recvmsg(
96
+ namelen,
97
+ socket.CMSG_LEN(2 * fd_model.itemsize),
98
+ )
99
+ assert len(ancdata) == 1
100
+
101
+ # The function's name will be in the "message" area of the recvmsg response.
102
+ # It will be populated with `namelen` bytes.
103
+ name = msg.decode('utf8')
104
+
105
+ # Two file descriptors are transferred to us from the database via the
106
+ # `sendmsg` protocol. These are for reading the input rows and writing
107
+ # the output rows, respectively.
108
+ fd0, fd1 = struct.unpack('<ii', ancdata[0][2])
109
+ ifile = os.fdopen(fd0, 'rb')
110
+ ofile = os.fdopen(fd1, 'wb')
111
+
112
+ # Keep receiving data on this socket until we run out.
113
+ while True:
114
+
115
+ # Read in the length of this row, a uint64. No data means we're done
116
+ # receiving.
117
+ buf = connection.recv(8)
118
+ if not buf:
119
+ break
120
+ length = struct.unpack('<q', buf)[0]
121
+ if not length:
122
+ break
123
+
124
+ # Map in the input shared memory segment from the fd we received via
125
+ # recvmsg.
126
+ mem = mmap.mmap(
127
+ ifile.fileno(),
128
+ length,
129
+ mmap.MAP_SHARED,
130
+ mmap.PROT_READ,
131
+ )
132
+
133
+ # Read row data
134
+ response_size = 0
135
+ out = io.BytesIO()
136
+
137
+ ifile.seek(0)
138
+ try:
139
+ # Run the function
140
+ asyncio.run(
141
+ app.call(
142
+ name,
143
+ io.BytesIO(ifile.read(length)),
144
+ out,
145
+ data_format='rowdat_1',
146
+ data_version='1.0',
147
+ ),
148
+ )
149
+
150
+ # Write results
151
+ buf = out.getbuffer()
152
+ response_size = len(buf)
153
+ ofile.truncate(max(128*1024, response_size))
154
+ ofile.seek(0)
155
+ ofile.write(buf)
156
+ ofile.flush()
157
+
158
+ # Complete the request by send back the status as two uint64s on the
159
+ # socket:
160
+ # - http status
161
+ # - size of data in output shared memory
162
+ connection.send(struct.pack('<qq', 200, response_size))
163
+
164
+ except Exception as exc:
165
+ errmsg = f'error occurred in executing function `{name}`: {exc}\n'
166
+ logger.error(errmsg.rstrip())
167
+ for line in traceback.format_exception(exc): # type: ignore
168
+ logger.error(line.rstrip())
169
+ connection.send(
170
+ struct.pack(
171
+ f'<qq{len(errmsg)}s', 500,
172
+ len(errmsg), errmsg.encode('utf8'),
173
+ ),
174
+ )
175
+ break
176
+
177
+ finally:
178
+ # Close the shared memory object.
179
+ mem.close()
180
+
181
+ # Close shared memory files.
182
+ ifile.close()
183
+ ofile.close()
184
+
185
+ # Close the connection
186
+ connection.close()
187
+
188
+
189
+ if __name__ == '__main__':
190
+ parser = argparse.ArgumentParser(
191
+ prog='python -m singlestoredb.functions.ext.mmap',
192
+ description='Run a collacated Python UDF server',
193
+ )
194
+ parser.add_argument(
195
+ '--max-connections', metavar='n', type=int, default=32,
196
+ help='maximum number of server connections before refusing them',
197
+ )
198
+ parser.add_argument(
199
+ '--single-thread', default=False, action='store_true',
200
+ help='should the server run in single-thread mode?',
201
+ )
202
+ parser.add_argument(
203
+ '--socket-path', metavar='file-path',
204
+ default=os.path.join(tempfile.gettempdir(), secrets.token_hex(16)),
205
+ help='path to communications socket',
206
+ )
207
+ parser.add_argument(
208
+ '--db', metavar='conn-str', default='',
209
+ help='connection string to use for registering functions',
210
+ )
211
+ parser.add_argument(
212
+ '--replace-existing', action='store_true',
213
+ help='should existing functions of the same name '
214
+ 'in the database be replaced?',
215
+ )
216
+ parser.add_argument(
217
+ '--log-level', metavar='[info|debug|warning|error]', default='info',
218
+ help='logging level',
219
+ )
220
+ parser.add_argument(
221
+ '--process-mode', metavar='[thread|subprocess]', default='subprocess',
222
+ help='how to handle concurrent handlers',
223
+ )
224
+ parser.add_argument(
225
+ 'functions', metavar='module.or.func.path', nargs='*',
226
+ help='functions or modules to export in UDF server',
227
+ )
228
+ args = parser.parse_args()
229
+
230
+ logger.setLevel(getattr(logging, args.log_level.upper()))
231
+
232
+ if os.path.exists(args.socket_path):
233
+ try:
234
+ os.unlink(args.socket_path)
235
+ except (IOError, OSError):
236
+ logger.error(f'could not remove existing socket path: {args.socket_path}')
237
+ sys.exit(1)
238
+
239
+ # Create application
240
+ app = asgi.create_app(
241
+ args.functions,
242
+ app_mode='collocated',
243
+ data_format='rowdat_1',
244
+ url=args.socket_path,
245
+ )
246
+
247
+ funcs = app.show_create_functions(replace=True) # type: ignore
248
+ if not funcs:
249
+ logger.error('no functions specified')
250
+ sys.exit(1)
251
+
252
+ for f in funcs:
253
+ logger.info(f'function: {f}')
254
+
255
+ # Register functions with database
256
+ if args.db:
257
+ logger.info('registering functions with database')
258
+ app.register_functions(args.db, replace=args.replace_existing) # type: ignore
259
+
260
+ # Create the Unix socket server.
261
+ server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
262
+
263
+ # Bind our server to the path.
264
+ server.bind(args.socket_path)
265
+
266
+ logger.info(f'using socket path: {args.socket_path}')
267
+
268
+ # Listen for incoming connections. Argument is the number of connections to
269
+ # keep in the backlog before we begin refusing them; 32 is plenty for this
270
+ # simple case.
271
+ server.listen(args.max_connections)
272
+
273
+ # Accept connections forever.
274
+ try:
275
+ while True:
276
+ # Listen for the next connection on our port.
277
+ connection, client_address = server.accept()
278
+
279
+ if args.process_mode == 'thread':
280
+ tcls = threading.Thread
281
+ else:
282
+ tcls = multiprocessing.Process # type: ignore
283
+
284
+ t = tcls(
285
+ target=_handle_request,
286
+ args=(app, connection, client_address),
287
+ )
288
+
289
+ t.start()
290
+
291
+ # NOTE: The following line forces this process to handle requests
292
+ # serially. This makes it easier to understand what's going on.
293
+ # In real life, though, parallel is much faster. To use parallel
294
+ # handling, just comment out the next line.
295
+ if args.single_thread:
296
+ t.join()
297
+
298
+ except KeyboardInterrupt:
299
+ sys.exit(0)
300
+
301
+ finally:
302
+ # Remove the socket file before we exit.
303
+ try:
304
+ os.unlink(args.socket_path)
305
+ except (IOError, OSError):
306
+ logger.error(f'could not remove socket path: {args.socket_path}')
@@ -5,7 +5,6 @@ import numbers
5
5
  import os
6
6
  import re
7
7
  import string
8
- import textwrap
9
8
  import typing
10
9
  from typing import Any
11
10
  from typing import Callable
@@ -16,7 +15,6 @@ from typing import Sequence
16
15
  from typing import Tuple
17
16
  from typing import TypeVar
18
17
  from typing import Union
19
- from urllib.parse import urljoin
20
18
 
21
19
  try:
22
20
  import numpy as np
@@ -611,8 +609,11 @@ def dtype_to_sql(dtype: str, default: Any = None) -> str:
611
609
 
612
610
  def signature_to_sql(
613
611
  signature: Dict[str, Any],
614
- base_url: Optional[str] = None,
612
+ url: Optional[str] = None,
615
613
  data_format: str = 'rowdat_1',
614
+ app_mode: str = 'remote',
615
+ link: Optional[str] = None,
616
+ replace: bool = False,
616
617
  ) -> str:
617
618
  '''
618
619
  Convert a dictionary function signature into SQL.
@@ -646,37 +647,27 @@ def signature_to_sql(
646
647
  host = os.environ.get('SINGLESTOREDB_EXT_HOST', '127.0.0.1')
647
648
  port = os.environ.get('SINGLESTOREDB_EXT_PORT', '8000')
648
649
 
649
- url = urljoin(base_url or f'https://{host}:{port}', signature['endpoint'])
650
+ if app_mode.lower() == 'remote':
651
+ url = url or f'https://{host}:{port}/invoke'
652
+ elif url is None:
653
+ raise ValueError('url can not be `None`')
650
654
 
651
655
  database = ''
652
656
  if signature.get('database'):
653
657
  database = escape_name(signature['database']) + '.'
654
658
 
655
- replace = 'OR REPLACE ' if signature.get('replace') else ''
659
+ or_replace = 'OR REPLACE ' if (bool(signature.get('replace')) or replace) else ''
656
660
 
657
- return (
658
- f'CREATE {replace}EXTERNAL FUNCTION {database}{escape_name(signature["name"])}' +
659
- '(' + ', '.join(args) + ')' + returns +
660
- f' AS REMOTE SERVICE "{url}" FORMAT {data_format.upper()};'
661
- )
662
-
663
-
664
- def func_to_env(func: Callable[..., Any]) -> str:
665
- # TODO: multiple functions
666
- signature = get_signature(func)
667
- env_name = signature['environment']
668
- replace = 'OR REPLACE ' if signature.get('replace') else ''
669
- packages = ', '.join(escape_item(x, 'utf8') for x in signature.get('packages', []))
670
- resources = ', '.join(escape_item(x, 'utf8') for x in signature.get('resources', []))
671
- code = inspect.getsource(func)
661
+ link_str = ''
662
+ if link:
663
+ if not re.match(r'^[\w_]+$', link):
664
+ raise ValueError(f'invalid LINK name: {link}')
665
+ link_str = f' LINK {link}'
672
666
 
673
667
  return (
674
- f'CREATE {replace}ENVIRONMENT {env_name} LANGUAGE PYTHON ' +
675
- 'USING EXPORTS ' + escape_name(func.__name__) + ' ' +
676
- (f'\n PACKAGES ({packages}) ' if packages else '') +
677
- (f'\n RESOURCES ({resources}) ' if resources else '') +
678
- '\n AS CLOUD SERVICE' +
679
- '\n BEGIN\n' +
680
- textwrap.indent(code, ' ') +
681
- ' END;'
668
+ f'CREATE {or_replace}EXTERNAL FUNCTION ' +
669
+ f'{database}{escape_name(signature["name"])}' +
670
+ '(' + ', '.join(args) + ')' + returns +
671
+ f' AS {app_mode.upper()} SERVICE "{url}" FORMAT {data_format.upper()}'
672
+ f'{link_str};'
682
673
  )
@@ -44,10 +44,10 @@ class Manager(object):
44
44
  """SingleStoreDB manager base class."""
45
45
 
46
46
  #: Management API version if none is specified.
47
- default_version = 'v1'
47
+ default_version = config.get_option('management.version')
48
48
 
49
49
  #: Base URL if none is specified.
50
- default_base_url = 'https://api.singlestore.com'
50
+ default_base_url = config.get_option('management.base_url')
51
51
 
52
52
  #: Object type
53
53
  obj_type = ''
@@ -65,11 +65,13 @@ def start_http_server(database, data_format='rowdat_1'):
65
65
  time.sleep(3)
66
66
  retries -= 1
67
67
 
68
- app = create_app(ext_funcs)
68
+ app = create_app(
69
+ ext_funcs,
70
+ url=f'http://{HTTP_HOST}:{port}/invoke',
71
+ data_format=data_format,
72
+ )
69
73
  app.register_functions(
70
- base_url=f'http://{HTTP_HOST}:{port}',
71
74
  database=database,
72
- data_format=data_format,
73
75
  )
74
76
 
75
77
  with s2.connect(database=database) as conn:
@@ -412,17 +412,6 @@ class TestUDF(unittest.TestCase):
412
412
  assert to_sql(foo) == '`hello``_``world`(`x` BIGINT NOT NULL) ' \
413
413
  'RETURNS BIGINT NOT NULL'
414
414
 
415
- # Add database name
416
- @udf(database='mydb')
417
- def foo(x: int) -> int: ...
418
- assert to_sql(foo) == '`mydb`.`foo`(`x` BIGINT NOT NULL) ' \
419
- 'RETURNS BIGINT NOT NULL'
420
-
421
- @udf(database='my`db')
422
- def foo(x: int) -> int: ...
423
- assert to_sql(foo) == '`my``db`.`foo`(`x` BIGINT NOT NULL) ' \
424
- 'RETURNS BIGINT NOT NULL'
425
-
426
415
  def test_dtypes(self):
427
416
  assert dt.BOOL() == 'BOOL NULL'
428
417
  assert dt.BOOL(nullable=False) == 'BOOL NOT NULL'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: singlestoredb
3
- Version: 1.0.2
3
+ Version: 1.0.3
4
4
  Summary: Interface to the SingleStoreDB database and workspace management APIs
5
5
  Home-page: https://github.com/singlestore-labs/singlestoredb-python
6
6
  Author: SingleStore
@@ -1,13 +1,13 @@
1
- _singlestoredb_accel.abi3.so,sha256=h_78j0iV0aEoVaA4uPmTME8DTqBKRDt_BAjUCgj1cfc,206109
2
- singlestoredb-1.0.2.dist-info/RECORD,,
3
- singlestoredb-1.0.2.dist-info/LICENSE,sha256=Mlq78idURT-9G026aMYswwwnnrLcgzTLuXeAs5hjDLM,11341
4
- singlestoredb-1.0.2.dist-info/WHEEL,sha256=_VEguvlLpUd-c8RbFMA4yMIVNMBv2LhpxYLCEQ-Bogk,113
5
- singlestoredb-1.0.2.dist-info/entry_points.txt,sha256=bSLaTWB5zGjpVYPAaI46MkkDup0su-eb3uAhCNYuRV0,48
6
- singlestoredb-1.0.2.dist-info/top_level.txt,sha256=SDtemIXf-Kp-_F2f_S6x0db33cHGOILdAEsIQZe2LZc,35
7
- singlestoredb-1.0.2.dist-info/METADATA,sha256=ZDy0IGkaKydgjvPH_QGR5D-hFy4cfIh4NiwKbkTCH20,5515
1
+ _singlestoredb_accel.abi3.so,sha256=JUOCyPIKYy6ozdXVp054_j5cs0OniL1iHW_ujIWXrgg,206109
2
+ singlestoredb-1.0.3.dist-info/RECORD,,
3
+ singlestoredb-1.0.3.dist-info/LICENSE,sha256=Mlq78idURT-9G026aMYswwwnnrLcgzTLuXeAs5hjDLM,11341
4
+ singlestoredb-1.0.3.dist-info/WHEEL,sha256=_VEguvlLpUd-c8RbFMA4yMIVNMBv2LhpxYLCEQ-Bogk,113
5
+ singlestoredb-1.0.3.dist-info/entry_points.txt,sha256=bSLaTWB5zGjpVYPAaI46MkkDup0su-eb3uAhCNYuRV0,48
6
+ singlestoredb-1.0.3.dist-info/top_level.txt,sha256=SDtemIXf-Kp-_F2f_S6x0db33cHGOILdAEsIQZe2LZc,35
7
+ singlestoredb-1.0.3.dist-info/METADATA,sha256=D-7UBUyQmH38miNEnUpaWgLmzIdEP-1uxhXxC8LJki8,5515
8
8
  singlestoredb/auth.py,sha256=u8D9tpKzrqa4ssaHjyZnGDX1q8XBpGtuoOkTkSv7B28,7599
9
- singlestoredb/config.py,sha256=ZfLiOeqADAUMekvA9OZoDWDLdvXTGebPOcmMnN33Drw,7476
10
- singlestoredb/__init__.py,sha256=A8RLgMfoXrQq6Px3VKkXKI08uBzCtZz5BA2WufmiL4M,1634
9
+ singlestoredb/config.py,sha256=Xaipos7C0bLTM2EBsuBXRuA4NI-AL3Attb7KC-DqlSg,7856
10
+ singlestoredb/__init__.py,sha256=qDpKRys2vXRJVjzkvHX3g5DCp_FFJ2bw3Kw9qhFXdr0,1634
11
11
  singlestoredb/types.py,sha256=FIqO1A7e0Gkk7ITmIysBy-P5S--ItbMSlYvblzqGS30,9969
12
12
  singlestoredb/connection.py,sha256=gDBIs3XgLOROVHtzMx9zmSYILc0aRVY7k8we3b4TxCw,44227
13
13
  singlestoredb/pytest.py,sha256=OyF3BO9mgxenifYhOihnzGk8WzCJ_zN5_mxe8XyFPOc,9074
@@ -28,13 +28,13 @@ singlestoredb/tests/test_results.py,sha256=wg93sujwt-R9_eJCgSCElgAZhLDkIiAo3qPkP
28
28
  singlestoredb/tests/test_fusion.py,sha256=UPaxXt5YNa3GS44l4oZfmUcq89YgN7EWWIbW_oCkYao,15100
29
29
  singlestoredb/tests/test_plugin.py,sha256=qpO9wmWc62VaijN1sJ97YSYIX7I7Y5C6sY-WzwrutDQ,812
30
30
  singlestoredb/tests/test_basics.py,sha256=rUfUGZ54xybvgp11XYWdqnUYMKa6VckB3XkX9LFnxRw,44180
31
- singlestoredb/tests/test_ext_func.py,sha256=P7XkV7aFNmN_GN8tt_riGmSIAZRNEcy-XfAlNsdPQWE,37330
31
+ singlestoredb/tests/test_ext_func.py,sha256=Q-ZOl7fn6XfiHpHgxLvaBi7KSVzIehbS-cthitXpe8g,37347
32
32
  singlestoredb/tests/test_connection.py,sha256=RiE_NATLYPiMR5jWaBlcP5YddwitS6dzOHOVVOVXCSI,50741
33
33
  singlestoredb/tests/test_ext_func_data.py,sha256=9Zb0Z1v-Yr0uOc97NJwPWuvJB49pLhzWKZtZWt-e7-Y,47693
34
34
  singlestoredb/tests/test_exceptions.py,sha256=tfr_8X2w1UmG4nkSBzWGB0C7ehrf1GAVgj6_ODaG-TM,1131
35
35
  singlestoredb/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
36
36
  singlestoredb/tests/test_management.py,sha256=P5I50_gt1VE5ja4CNVo0j10fmwAqE57psxCvI_RWRFI,28223
37
- singlestoredb/tests/test_udf.py,sha256=Vqg5dt0Q1pLfhhUa6Jp0mYKQVs2CbLix2vy-XK70rRs,28490
37
+ singlestoredb/tests/test_udf.py,sha256=2Ml6VMTKIfstB-L31uX-zftwPsT5C64M29WZ6iuKdjI,28075
38
38
  singlestoredb/tests/test_http.py,sha256=RXasTqBWRn__omj0eLFTJYIbZjd0PPdIV2d4Cqz0MC8,8580
39
39
  singlestoredb/tests/utils.py,sha256=76eNdYFVnsw6S3J_RaGgGQ87Rlm8pxwyYaFYXnvAEvk,4673
40
40
  singlestoredb/tests/test_config.py,sha256=63lyIQ2KrvGE6C9403B_4Mc90mX4tp42ys5Bih2sXrE,11184
@@ -50,7 +50,7 @@ singlestoredb/management/__init__.py,sha256=jXtKvpvl5diiotXPiEi2EpJwhPLEMb4_MTpn
50
50
  singlestoredb/management/utils.py,sha256=ZFt1jaUwNij16UplNxK_kvvd-w54OunsBaGHojqqUn8,8329
51
51
  singlestoredb/management/cluster.py,sha256=_TT4tV43VPDrtcdS_VN-TTYij7yFQzjAMeuYRF9zKj8,14362
52
52
  singlestoredb/management/workspace.py,sha256=xIdSA_X8aiLnCZjNJg8-xYZeVyVeL1i4sVJUx-zMG_g,51907
53
- singlestoredb/management/manager.py,sha256=mxwW1OQFvUIaPzTo_-9Oolv4Xwvf3mL5VS3qnvR1Ydg,8764
53
+ singlestoredb/management/manager.py,sha256=m8I5zTmEqjMCEE4fmmVdzza8TvofhnIHvO0np0WH-Y8,8810
54
54
  singlestoredb/management/billing_usage.py,sha256=9ighjIpcopgIyJOktBYQ6pahBZmWGHOPyyCW4gu9FGs,3735
55
55
  singlestoredb/utils/config.py,sha256=m3Xn6hsbdKyLufSnbokhFJ9Vfaz9Qpkj1IEnIiH9oJQ,24503
56
56
  singlestoredb/utils/results.py,sha256=cqFK4-0CBSDcT-R1ixKIWN5_sCn9s9SoEO6Gllj8mCI,5204
@@ -100,13 +100,14 @@ singlestoredb/mysql/tests/thirdparty/test_MySQLdb/__init__.py,sha256=bG2vIeDuGH5
100
100
  singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py,sha256=t_OzqsVnj_ReBbmY_wx51ZcWbLz9nASZ0hno-9YeiyQ,8022
101
101
  singlestoredb/mysql/tests/thirdparty/test_MySQLdb/capabilities.py,sha256=AgEdvx7Njz_Y7KDMeQPMYI7y4nJRKblocVrC0VxVZZE,10171
102
102
  singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py,sha256=szE4Zodgf7YwhkMBOrCvUwhTWppVtaodsqlV-vJ7fmY,3090
103
- singlestoredb/functions/decorator.py,sha256=feFPevL6-3wE9KJCb3vrgqtSoY8Gc_lD0mgW242Rp3g,7524
103
+ singlestoredb/functions/decorator.py,sha256=H12MUeBw8VOppx6esntaR43ukeIffbnAr716CBpYJ4g,5193
104
104
  singlestoredb/functions/__init__.py,sha256=WL1LqgMTdnGOse3tQqmD-HH8TdfCPS89GNO7hO0v_aw,41
105
105
  singlestoredb/functions/dtypes.py,sha256=iP3_AvE2jBxlkziOHzoUvTtYCdBZlaxJHNgvGwp07Ao,36712
106
- singlestoredb/functions/signature.py,sha256=iqSsx8N8lU0ycNBO05HLAixy2OGIjrq33ntLME4YPmA,19343
107
- singlestoredb/functions/ext/asgi.py,sha256=DOf775i708hpUL52FQ1P_R5ORXHQjH9dbErUJ_L98jE,18911
106
+ singlestoredb/functions/signature.py,sha256=fNnlTfc0R0sM9wm78UwG7Ok9eMJTtOfawrIpjts2wdY,18866
107
+ singlestoredb/functions/ext/asgi.py,sha256=3Rp0m2DNf5yzbCQpaazTTVoDCPfp3jT6dJ7MlvHijw0,21938
108
108
  singlestoredb/functions/ext/arrow.py,sha256=WB7n1ACslyd8nlbFzUvlbxn1BVuEjA9-BGBEqCWlSOo,9061
109
109
  singlestoredb/functions/ext/__init__.py,sha256=kGCV3QC5pL95TytpI8pwvSVCqqoTrV8duQQEUp65sy4,66
110
+ singlestoredb/functions/ext/mmap.py,sha256=lvdKiGPh-H7LfkrYbPvcH5BWv9zuz7t2FAvW-nYdWzI,9759
110
111
  singlestoredb/functions/ext/json.py,sha256=UuUxTzlr5ztAbXqOGaVGUhO7xFN_oBY75nFh9B8cRog,10372
111
112
  singlestoredb/functions/ext/rowdat_1.py,sha256=yZElsItSbVTFlXU3N-ee6QIybxuksqwv1UE3Y6h45c0,22274
112
113
  singlestoredb/alchemy/__init__.py,sha256=dXRThusYrs_9GjrhPOw0-vw94in_T8yY9jE7SGCqiQk,2523