motifapi 0.2.0__tar.gz

motifapi-0.2.0/MANIFEST.in

@@ -0,0 +1 @@
+include motifapi/server.crt

motifapi-0.2.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: motifapi
+Version: 0.2.0
+Summary: Python interface to Motif recording systems
+Author: John Stowers
+Author-email: john@loopbio.com
+License: BSD
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.6
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: license
+Dynamic: requires-python
+Dynamic: summary
+
+Python interface to Motif recording systems

motifapi-0.2.0/README.md

@@ -0,0 +1,610 @@
+API for Motif Recording Systems (Python Wrapper)
+================================================
+
+[This library](https://github.com/loopbio/python-motifapi) allows you to control cameras
+and recording on loopbio [motif](http://loopbio.com/recording) recording systems, including
+the ability to control outputs and schedule operations. This allows you to implement
+experimental or operational protocols like;
+
+ * "start recording every hour for 30 minutes. while recording give this stimulus (switch this output)
+    every 5 minutes"
+ * "record at 9am and 3pm. at 5pm copy all recorded videos to network storage"
+ * "while recording, at minute 1 switch this output on, then every 30 seconds thereafter, switch on and
+    off this other output"
+
+**Table of Contents**
+
+ * [Examples](#examples)
+ * [Examples (IO and Scheduling)](#scheduling-examples)
+ * [API Documentation](#api-documentation)
+ * [Experiment Script Messages](#experiment-script-messages)
+ * [Scheduling](#scheduling-function)
+ * [API Documentation (Scheduling)](#scheduling-api-documentation)
+ * [MATLAB](#matlab) and [Other Language](#other-languages) support
+ * [Realtime Streaming](#realtime-streaming)
+
+## Getting Started
+
+* this library works out of the box on Python2 and Python3 and has no dependencies outside
+  of the standard library
+* this library wraps the REST+JSON API and handles sending and parsing the responses
+* this library is compatible with all motif versions, although real-time image streaming and
+  some IO operations are only supported in motif 5 and above
+* to control a motif recording system you need to know it's `IP_ADDRESS` and an `API_KEY`.
+  * the IP address is that which you browse to in order to control the recording software
+    using your web browser
+  * the API key is specific to the installation of the recording software and secures the
+    API from unauthorized access. To find the API key for the recording system you wish to
+    control, SSH into the machine (or request your sysadmin to) and execute the following
+    command `recnode-apikey`
+
+
+### Examples
+
+Set up the connection to the machine
+
+```python
+from motifapi import MotifApi
+
+IP_ADDRESS = '10.11.12.23'
+API_KEY = 'abcdef123456abcdef123456abcdef12'
+
+api = MotifApi(IP_ADDRESS, API_KEY)
+# check the client is connected
+api.call('version')
+# list the connected cameras
+api.call('cameras')
+```
+
+To start recording on all cameras with the predefined compression settings named 'high'
+(see the web UI for the names of your configured compression formats). The recording will run
+for 5 seconds and will have have the following metadata attached.
+
+```python
+api.call('recording/start',
+         codec='high',
+         duration=5,
+         metadata={'foo': 1, 'bar': 'bob'})
+```
+
+To stop all recordings on all cameras
+
+```python
+api.call('recording/stop')
+```
+
+To configure one camera and start recording on it alone
+
+```python
+api.call('camera/FAKE0/configure', Gain=0))
+api.call('camera/FAKE0/recording/start', codec='low')
+```
+
+Cameras are identified by their serial number ('FAKE0' in the example above). To find the serial
+numbers of connected cameras do
+
+```python
+api.call('cameras')
+```
+
+To find the status of a camera (is it recording, uploading, etc) do
+
+```python
+api.call('camera/FAKE0')
+```
+
+**See `examples/*.py` for further examples of API usage**
+
+## API Documentation
+
+The complete list of API endpoints/paths, as passed to `call(path, **arguments)`,
+can be found below. Text between `<` and `>` characters should be replaced
+with the appropriate values. Arguments are passed after the path, e.g.
+`api.call('recording/start', duration=5.0)`.
+
+ * `version`
+    * return the current software version
+ * `cameras`
+    * return a list of connected cameras, their names and serial numbers
+ * `camera/<serial>`
+    * returns the selected camera status
+    * `serial`: the serial number of the camera
+ * `camera/<serial>/configure`
+    * change camera configuration
+    * `serial`: the serial number of the camera
+    * arguments (for example, optional)
+        * `AcquisitionFrameRate`: change the framerate of the camera
+        * `ExposureTime`: exposure time in us
+        * `...` or any other camera supported parameter name and value
+ * `cameras/configure`
+    * as previous, but apply the configuration changes to every attached camera
+ * `camera/<serial>/read/<parameter_name>`
+    * change camera configuration
+    * `serial`: the serial number of the camera
+    * `parameter_name`: the name of a camera parameter, like that passed to `/configure`, e.g. `AcquisitionFrameRate`
+ * `cameras/read/<parameter_name>`
+    * as previous, but return values for all cameras
+ * `camera/<serial>/recording/start`
+    * start recording on the selected camera
+    * `serial`: the serial number of the camera
+    * arguments
+        * `filename` (optional): recording filename (excluding timestamp)
+        * `record_to_store` (optional)
+             * `True`
+             * `False`
+             * no argument provided: use configured defaults
+        * `codec` (optional): codec, or use configured default if omitted
+        * `duration` (optional): number of seconds to record for, or indefinately if omitted
+        * `metadata` (optional): a dictionary of metadata to save in the resulting video
+        * `codec_identifier` (optional): advanced use only
+ * `recording/start`
+    * as previous, but start recording on all cameras
+ * `camera/<serial>/recording/stop`
+    * stop recording on the selected camera
+    * `serial`: the serial number of the camera
+ * `recording/stop`
+    * as previous, but stop recording on all cameras
+ * `camera/<serial>/recordings`
+    * return a list of recordings
+    * `serial`: the serial number of the camera
+ * `recordings`
+    * as previous, but return recordings for all cameras
+ * `camera/<serial>/recordings/copy_all`
+    * copy (or move) all currently completed recordings to another location
+    * `serial`: the serial number of the camera
+    * arguments
+        * `location` (optional): local user path or if omitted, default configued location
+        * `delete_after` (optional, default=False): delete original recordings after successful copy
+        * `loopy_username` (optional)
+        * `loopy_url` (optional)
+        * `loopy_api_key` (optional)
+        * `loopy_import_base` (optional):
+          connection details of accesibly self hosted loopy instance for automatic
+          importing of videos into loopy after copy finishes
+ * `recordings/copy_all`
+    * as previous, but copy recordings from all cameras
+ * `camera/<serial>/recordings/export_all`
+    * export image stores to normal mp4 videos (or image stores)
+    * `serial`: the serial number of the camera
+    * arguments
+        * `to_store` (optional)
+            * `True`
+            * `False`
+            * no argument provided: use configured defaults
+       * `codec` (optional)
+       * `delete_after` (optional)
+       * `path` (optional)
+ * `recordings/export_all`
+    * as previous, but export image stores from all cameras
+ * `camera/<serial>/io/<name>/set`
+    * set named output associated with camera to provided value
+    * `serial`: the serial number of the camera
+    * `name`: the name of the configured output channel
+    * arguments
+        * `value`: the value to set on the output (backend dependent)
+ * `io/<name>/set`
+    * set named output to the provided value. in index or master mode in
+      a multiple camera setup, the named output channel must be on a
+      output device attached to the master or index node.
+      setup (or to the only attached camera in a single camera setup) 
+    * `name`: the name of the configured output channel
+    * arguments
+        * `value`: the value to set on the output (backend dependent)
+ * `io/<io_serial>/<io_port>/set` (DEPRECATED)
+    * configure and set output to provided value
+    * `io_serial`: serial number of IO device
+    * `io_port`: port on device to set
+    * arguments
+        * `value`: continuous value to set OR
+        * `state`: 0 or 1 to turn on or off
+ * `io/log`
+    * send multiple arguements to be recorded in the imgstore extra_data - in addition to the current frame_number and time of the log message
+ * `camera/<serial>/experiment/message`
+    * send a user-facing status message from an experiment script to the web UI
+      for the specified camera
+    * `serial`: the serial number of the camera
+    * arguments
+        * `message`: the text to display. pass an empty string to clear the message
+ * `experiment/message`
+    * as previous, but in a single camera setup or to target the multicam
+      master/index UI
+ * `multicam/synchronize`
+    * synchronize the cameras (in a multiple camera setup)
+ * `multicam/connect_camera/<serial>`
+    * start the camera process for the camera with the provided serial
+    * `serial`: the serial number of the camera.
+        * passing the special value `__all` here connects all cameras
+ * `multicam/disconnect_camera/<serial>`
+    * stops the camera process for the camera with the provided serial
+    * `serial`: the serial number of the camera.
+        * passing the special value `__all` here disconnects all cameras
+
+
+**Outputs and Toggling Values**
+
+There exists a special value for (digital) outputs that results in the output state being
+toggled (alternating) between minimum and maximum values. If you pass `value=+inf` then this means
+start with a switch to maximum, then toggle between minimum and maximum subsequently. `value=-inf`
+start with a switch to minimum, then toggle between maximum and minimum subsequently.
+
+## Experiment Script Messages
+
+Experiment scripts can send a short text message back to the Motif web UI using the
+`send_message` convenience method (or the `experiment/message` endpoint). The message
+appears in the Experiment Scripts section of the UI with a timestamp, and can be updated
+at any time to show progress or status.
+
+```python
+from motifapi import MotifApi
+
+api = MotifApi(IP_ADDRESS, API_KEY)
+
+# send a message (appears in the multicam or single-cam UI)
+api.send_message('Starting trial 1')
+
+# send a message targeting a specific camera
+api.send_message('Acquiring baseline', serial='FAKE0')
+
+# clear the message
+api.send_message('')
+```
+
+Using `api.call` directly:
+
+```python
+# to the multicam / single-cam UI
+api.call('experiment/message', message='Hello from script')
+
+# to a specific camera's UI
+api.call('camera/FAKE0/experiment/message', message='Running phase 2')
+```
+
+## Scheduling Function
+
+Motif allows scheduling of almost all previously documented API operations.
+Scheduling allows executing a certain operation as a defined time. 
+
+This allows for example, to schedule recordings and their subsequent copy
+to storage to occur at specific times. Task scheduling re-uses [Cron syntax]()
+with some extensions. 
+
+### Cron Syntax
+
+Please refer to [here](README.cronex.md) for a complete specification and more examples.
+
+Each scheduled operation is specified with a combination of six white-space
+separated fields that dictate when the event should occur. In order, the fields
+specify trigger times for the second, minute, hour, day of the month,
+month and day of the week.
+
+```text
+    .------------------ Second (0 - 59)
+    |   .--------------- Minute (0 - 59)
+    |   |   .------------ Hour (0 - 23)
+    |   |   |   .--------- Day of the month (1 - 31)
+    |   |   |   |   .------ Month (1 - 12) or Jan, Feb ... Dec
+    |   |   |   |   |   .---- Day of the week (0 (Sun.; also 7) - 6 (Sat.))
+    |   |   |   |   |   |
+    V   V   V   V   V   V
+    *   *   *   *   *   *
+```
+
+### Scheduling API Documentation
+
+The API for scheduling tasks is very similar to that of other tasks - API endpoints are
+prefixed with `schedule/VERB` may/should be passed additional optional or
+compulsory arguments.
+
+**Scheduling specific arguments**
+
+   * `task_name` (required): A unique short identifier for this task
+   * `cron_expression` (required): a cron specifier conforming to the cron syntax above
+   * `camera_relative` (optional, Motif 5 and above only)
+      * `True`: if true all absolute and relative dates are relative to the start of the recording
+      * `False` (default): all dates are absolute, and monotonic expressions are relative to 1-Jan-1970
+
+**Camera Relative Schedules**
+
+Please ensure you have read the [full documentation](README.cron.md) on cron triggers first. Often one needs to schedule events not in the conventional cron sense - relative to the date and time of the day, but instead wants to schedule operations relative to when a recording was started. If a task is scheduled with `camera_relative=True` then dates are interpreted differently. For example the expression
+
+ * `0 5 * ? * * *`
+    * with `camera_relative=False` means execute the task *at second :00 of minute :05 of every hour*
+        * e.g. Fri May 17 16:05:00 UTC 2019, Fri May 17 17:05:00 UTC 2019, Fri May 17 18:05:00 UTC 2019
+    * with `camera_relative=False` means execute the task *at second :00 of minute :05 of every hour* relative to when recording was started.
+        * e.g., for a recording started at Fri May 17 13:14:00 UTC 2019, the task would trigger at Fri May 17 13:19:00 UTC 2019, Fri May 17 14:19:00 UTC 2019, etc.
+
+With monotonic triggers `7%7` or `%10` this can be confusing. Monotonic triggers execute *every unit of time*. For example the expression
+
+ * `0 * %2 ? * * *`
+    * with `camera_relative=False` means *execute every 2 hours starting at 00:00:00*
+        * e.g. Fri May 17 00:00:00 UTC 2019, Fri May 17 02:00:00 UTC 2019
+    * with `camera_relative=True` means *execute every 2 hours starting at the time of recording start*
+        * e.g., for a recording started at Fri May 17 13:14:00 UTC 2019, the task would trigger at Fri May 17 15:14:00 UTC 2019, Fri May 17 17:14:00 UTC 2019, etc.
+
+**API (continued)**
+
+ * `schedule`
+    * list all scheduled tasks, return also including the current time on
+      the system
+ * `schedule/clear`
+    * delete all scheduled tasks
+ * `schedule/<task_name>/clear`
+    * delete the provided task
+    * `task_name`: the identifier of the task to clear 
+ * `schedule/camera/<serial>/recording/start`
+    * schedule the start of recording on the selected camera
+    * `serial`: the serial number of the camera
+    * arguments
+        * scheduling specific (above), excluding `camera_relative`
+        * see `recording/start` (above)
+ * `schedule/recording/start`
+    * as previous, but schedule the start of recording on all cameras
+ * `schedule/camera/<serial>/recordings/copy_all`
+    * schedule copy (or move) of recordings
+    * `serial`: the serial number of the camera
+    * arguments
+        * scheduling specific (above), excluding `camera_relative`
+        * see `recordings/copy_all` (above)
+ * `schedule/recordings/copy_all`
+    * as previous, but schedule copy (or move) of recordings on all cameras
+ * `schedule/camera/<serial>/recordings/export_all`
+    * schedule export of recordings on the selected camera
+    * `serial`: the serial number of the camera
+    * arguments
+        * scheduling specific (above), excluding `camera_relative`
+        * see `recordings/export_all`
+ * `schedule/recordings/export_all`
+    * as previous, but schedule export of recordings on all cameras
+ * `schedule/camera/<serial>/configure/<name>`
+    * schedule the change of a camera configuration
+    * `serial`: the serial number of the camera
+    * `name`: the the name of the parameter to change (e.g. `ExposureTime`)
+    * arguments
+        * scheduling specific (above)
+        * `value` (required): the new value of the parameter
+ * `schedule/cameras/configure/<name>`
+    * as previous, but schedule the change of camera configuration on all cameras
+ * `schedule/camera/<serial>/io/<name>/set`
+    * schedule the setting of a named output, on an IO device connected to the
+      supplied camera, to a provided value
+    * `serial`: the serial number of the camera
+    * `name`: the name of the configured output channel
+    * arguments
+        * scheduling specific (above)
+        * `value` (required): the new value of the parameter
+ * `schedule/io/<name>/set`
+    * schedule the setting of the named output to the provided value.
+      in index or master mode in a multiple camera setup, the named output
+      channel must be on a output device attached to the master or index node.
+    * arguments
+        * scheduling specific (above)
+        * `value` (required): the new value of the parameter
+
+## Scheduling Examples
+
+To schedule a 30 minute recording as configured above to be executed every-hour-on-the-hour between
+9am and 4pm, make the following API call
+
+```python
+api.call('schedule/recording/start',
+         task_name='record_video',
+         cron_expression='0 6-16 * * * *',
+         duration=30*60)
+```
+
+To schedule events relative to the start of recording on a specific camera, consider the following examples.
+
+```python
+import datetime
+from motifapi import datetime_to_cron
+
+camera_serial = '123456'
+io_name = 'led'
+
+# note the use of datetime_to_cron() which converts an absolute date/time to a non-repeating
+# non-monotonic cron expression. Furthermore, note that because this should occur 17s relative
+# to the start of recording, we pass a datetime initialized from 0 seconds.
+
+api.call('schedule/camera/%s/io/%s/set' % (camera_serial, io_name),
+         task_name='led_on_after_17s',
+         cron_expression=datetime_to_cron(datetime.datetime.fromtimestamp(0) + datetime.timedelta(seconds=17)),
+         camera_relative=True,
+         value=1.0)
+```
+
+monotonic expressions do not require the use of `datetime_to_cron` as the repeat regardless. For example the following expression flashes / toggles an LED every second
+
+```python
+api.call('schedule/camera/%s/io/%s/set' % (camera_serial, io_name),
+         task_name='toggle_led_2s',
+         cron_expression='%2 * * ? * * *',
+         camera_relative=True,
+         value=float('+inf'))
+```
+
+neither to regular repeating cron expressions. Consider the following two scheduled operations that turn on an LED at `00:15:00` and `00:45:00` and turn it off at `00:00:00` and `00:30:00`.
+
+```python
+api.call('schedule/camera/%s/io/%s/set' % (camera_serial, io_name),
+         task_name='turn_on_led',
+         cron_expression='0 15,45 * ? * * *',
+         camera_relative=True,
+         value=1.0)
+api.call('schedule/camera/%s/io/%s/set' % (camera_serial, io_name),
+         task_name='turn_off_led',
+         cron_expression='0 0,30 * ? * * *',
+         camera_relative=True,
+         value=0.0)
+```
+
+note: this same sequence of operations could have likewise been described using a monotonic expression `0 15%15 * ? * * *` and 'toggling IO (see above)' `value=float(+inf)`.
+
+
+## FAQ
+
+ * Q) What if Motif is running on another computer? how to I find the time on the system
+
+```python
+now = api.call('schedule')['now']
+dt = datetime.datetime.fromtimestamp(now)
+```
+
+ * Q) What is the easiest way to schedule something to happen 'some time from now'
+ 
+```python
+# following from the example above
+import datetime
+from motifapi import datetime_to_cron
+future = dt + datetime.timedelta(seconds=30)
+api.call('schedule/recording/start',
+         task_name='record_video_in_30s_time',
+         cron_expression=datetime_to_cron(future),
+         duration=30*60)
+```
+ 
+ * Q) Cron expressions are confusing. Can you recommend ways to help understanding
+    * you can test and design cron expressions online using the following site (note: neither support monotonic expressions)
+        * [https://www.freeformatter.com/cron-expression-generator-quartz.html](https://www.freeformatter.com/cron-expression-generator-quartz.html) 
+        * [https://cronexpressiondescriptor.azurewebsites.net/](https://cronexpressiondescriptor.azurewebsites.net/) 
+
+**See examples/scheduler.py for more information**
+
+
+## Other Languages
+
+The API is a pure REST api, which means any language / framework / environment with support for calling REST+JSON endpoints and handling their responses is supported.
+
+Note: You must provide the HTTPS server certificate and authentication API key in a manner appropriate for your language / framework / environment.
+
+### MATLAB
+
+Because the Motif API is pure HTTP+JSON you can also use MATLAB to control the recording systems. Note however the following caveats
+
+* Examples below are for the hypothetical IP address `10.11.12.13` and API_KEY `abcdef123456abcdef123456abcdef12`. Please replace with values appropriate to your system
+  * On windows, you can find the API key in the config file (link in Start menu), on Linux by executing `recnode-apikey` on the PC
+* `server.crt` must be the full path to the `server.crt` file which you should [download](https://raw.githubusercontent.com/loopbio/python-motifapi/master/motifapi/server.crt) from this repository. 
+* Not all endpoints are `POST`, although most are. `GET` endpoints are read using `webread`, `POST`, `PATCH`, `DELETE` using `webwrite`. The endpoint type is documented in [api.py](http://github.com/loopbio/python-motifapi/blob/master/motifapi/api.py).
+
+**Checking Status**
+
+```matlab
+% example showing how to list connected cameras from MATLAB
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'})
+# list the connected cameras
+webread('https://10.11.12.13:6083/api/1/cameras', o)
+```
+
+**Turning on an Output**
+
+```matlab
+% example showing how to set an output using a POST request
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'}, ...
+               'MediaType', 'application/json')
+arguments = struct('value',0.3);
+webwrite('https://10.11.12.13:6083/api/1/io/led/set',arguments,o)
+```
+
+**Starting Recording**
+
+```matlab
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'}, ...
+               'MediaType', 'application/json')
+arguments = struct('codec','h264');
+webwrite('https://10.11.12.13:6083/api/1/recording/start',arguments,o)
+```
+
+**Stopping Recording**
+
+```matlab
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'}, ...
+               'MediaType', 'application/json')
+arguments = struct('codec','h264');
+webwrite('https://10.11.12.13:6083/api/1/recording/stop',arguments,o)
+```
+
+**Checking Status**
+
+```matlab
+% example showing how to list connected cameras from MATLAB
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'})
+# list the connected cameras
+webread('https://10.11.12.13:6083/api/1/cameras', o)
+```
+
+**Turning on an Output**
+
+```matlab
+% example showing how to set an output using a POST request
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'}, ...
+               'MediaType', 'application/json')
+arguments = struct('value',0.3);
+webwrite('https://10.11.12.13:6083/api/1/io/led/set',arguments,o)
+```
+
+**Starting Recording**
+
+```matlab
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'}, ...
+               'MediaType', 'application/json')
+arguments = struct('codec','h264');
+webwrite('https://10.11.12.13:6083/api/1/recording/start',arguments,o)
+```
+
+**Stopping Recording**
+
+```matlab
+o = weboptions('CertificateFilename', 'server.crt', ...
+               'HeaderFields', {'X-Api-Key', 'abcdef123456abcdef123456abcdef12'}, ...
+               'MediaType', 'application/json')
+arguments = struct('codec','h264');
+webwrite('https://10.11.12.13:6083/api/1/recording/stop',arguments,o)
+```
+
+<!---motifcutstart--->
+
+## Realtime Streaming
+
+Motif can, with very low latency (&lt;1ms), stream realtime images from the camera, without interfering with
+the recording, compression, or any other motif functions. This allows developing *out of process* realtime
+image processing algorithms for closed loop experiments. Such experiments could for example then provide
+stimulus to the animal using either Motif connected and configured outputs, or other user provided methods.
+
+Per default streaming is limited to localhost (so such scripts must run on the same PC as Motif), however
+Motif can be configured to stream also to other network locations if you are aware of the latency
+implications and have sufficient network infrastructure.
+
+### Streaming Images
+
+A realtime image stream can be established as follows (after constructing the `api` object with the correct IP address and API_KEY as indicated above
+
+```python
+# small example using opencv to show the image in realtime, outside of motif
+
+stream = api.get_stream(stream_type=MotifApi.STREAM_TYPE_IMAGE)
+if stream is not None:
+    while True:
+        I, md = stream.get_next_image()
+        cv2.imshow('live', I)
+        cv2.waitKey(1)
+```
+
+### Streaming State
+
+If you have other more custom data acquisition needs not supported by Motif IO / DAQ support, and want to subsequently 
+integrate or synchronize this custom data with the motif imagestore `frame_number`s and `frame_time`s then you can use the realtime state streaming. Motif publishes to this stream, at low latency (&lt;&lt;1ms), the current `frame_number`, `frame_time`, and other information. A realtime state stream is established as follows
+
+```python
+stream = api.get_stream(stream_type=MotifApi.STREAM_TYPE_STATE)
+if stream is not None:
+    while True:
+        print stream.get_next_state()
+```
+
+<!---motifcutend--->

motifapi-0.2.0/motifapi/__init__.py

@@ -0,0 +1,5 @@
+__version__ = '0.2.0'
+
+from .api import MotifError, MotifApiError, MotifApi
+from .schedule import datetime_to_cron
+from .util import get_experiment_metadata

motifapi-0.2.0/motifapi/api.py

@@ -0,0 +1,328 @@
+import re
+import json
+import ssl
+import socket
+import os.path
+import subprocess
+import logging
+import urllib.request
+import urllib.error
+import http.client
+
+DEFAULT_HTTP_TIMEOUT = 10  # seconds
+
+
+# http://code.activestate.com/recipes/577548-https-httplib-client-connection-with-certificate-v/
+# http://stackoverflow.com/questions/1875052/using-paired-certificates-with-urllib2
+
+class HTTPSClientAuthHandler(urllib.request.HTTPSHandler):
+    """
+    Allows sending a client certificate with the HTTPS connection.
+    This version also validates the peer (server) certificate.
+    """
+
+    def __init__(self, key=None, cert=None, ca_certs=None):
+        self.ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+        self.ctx.check_hostname = False
+        if ca_certs:
+            self.ctx.verify_mode = ssl.CERT_REQUIRED
+            self.ctx.load_verify_locations(ca_certs)
+        else:
+            self.ctx.verify_mode = ssl.CERT_NONE
+        if cert:
+            self.ctx.load_cert_chain(cert, key)
+        urllib.request.HTTPSHandler.__init__(self, context=self.ctx)
+
+    def https_open(self, req):
+        return self.do_open(self.get_connection, req)
+
+    def get_connection(self, host, timeout=DEFAULT_HTTP_TIMEOUT):
+        return HTTPSConnection(host, timeout=timeout, context=self.ctx)
+
+
+class HTTPSConnection(http.client.HTTPSConnection):
+    """
+    Overridden to allow peer certificate validation using an SSLContext.
+    """
+
+    def __init__(self, host, **kwargs):
+        self.ctx = kwargs.pop('context', None)
+        http.client.HTTPSConnection.__init__(self, host, **kwargs)
+
+    def connect(self):
+        sock = socket.create_connection((self.host, self.port), self.timeout)
+
+        if self._tunnel_host:
+            self.sock = sock
+            self._tunnel()
+
+        if self.ctx:
+            self.sock = self.ctx.wrap_socket(sock, server_hostname=self.host)
+        else:
+            self.sock = sock
+
+
+class MethodRequest(urllib.request.Request):
+    # See: https://gist.github.com/logic/2715756
+
+    def __init__(self, *args, **kwargs):
+        if 'method' in kwargs:
+            self._method = kwargs['method']
+            del kwargs['method']
+        else:
+            self._method = None
+        return urllib.request.Request.__init__(self, *args, **kwargs)
+
+    def get_method(self, *args, **kwargs):
+        return self._method if self._method is not None else urllib.request.Request.get_method(self, *args, **kwargs)
+
+
+class MotifError(Exception):
+    pass
+
+
+class MotifApiError(MotifError):
+    def __init__(self, message, code):
+        self.code = code
+        Exception.__init__(self, message)
+
+
+class MotifApi(object):
+
+    STREAM_TYPE_IMAGE = 1
+    STREAM_TYPE_STATE = 2
+
+    API = {r'version$': 'GET',
+           r'cameras$': 'GET',
+           r'cameras/configure$': 'PATCH',
+           r'cameras/read/(?P<name>[^\s /]+)$': 'GET',
+           r'camera/(?P<serial>[^\s /]+)$': 'GET',
+           r'camera/(?P<serial>[^\s /]+)/configure$': 'PATCH',
+           r'camera/(?P<serial>[^\s /]+)/read/(?P<name>[^\s /]+)$': 'GET',
+           r'camera/(?P<serial>[^\s /]+)/recording/start$': 'POST',
+           r'camera/(?P<serial>[^\s /]+)/recording/stop$': 'POST',
+           r'camera/(?P<serial>[^\s /]+)/recordings$': 'GET',
+           r'camera/(?P<serial>[^\s /]+)/recordings/copy_all$': 'POST',
+           r'camera/(?P<serial>[^\s /]+)/recordings/export_all$': 'POST',
+           r'camera/(?P<serial>[^\s /]+)/io/(?P<name>[^\s /]+)/set': 'POST',
+           r'camera/(?P<serial>[^\s /]+)/io/log': 'POST',
+           r'camera/(?P<serial>[^\s /]+)/io/read': 'GET',
+           r'recording/start$': 'POST',
+           r'recording/stop$': 'POST',
+           r'recordings$': 'GET',
+           r'recordings/copy_all$': 'POST',
+           r'recordings/export_all$': 'POST',
+           r'schedule$': 'GET',
+           r'schedule/clear$': 'POST',
+           r'schedule/(?P<identifier>[^\s /]+)/clear$': 'DELETE',
+           r'schedule/recording/start$': 'POST',
+           r'schedule/camera/(?P<serial>[^\s /]+)/recording/start$': 'POST',
+           r'schedule/recordings/copy_all': 'POST',
+           r'schedule/camera/(?P<serial>[^\s /]+)/recordings/copy_all$': 'POST',
+           r'schedule/recordings/export_all': 'POST',
+           r'schedule/camera/(?P<serial>[^\s /]+)/recordings/export_all$': 'POST',
+           r'schedule/io/(?P<name>[^\s /]+)/set': 'POST',
+           r'schedule/camera/(?P<serial>[^\s /]+)/io/(?P<name>[^\s /]+)/set': 'POST',
+           r'schedule/cameras/configure/(?P<name>[^\s /]+)': 'POST',
+           r'schedule/camera/(?P<serial>[^\s /]+)/configure/(?P<name>[^\s /]+)': 'POST',
+           r'io/(?P<io_serial>[^\s /]+)/(?P<io_port>[^\s /]+)/set': 'POST',
+           r'io/(?P<name>[^\s /]+)/set': 'POST',
+           r'io/log': 'POST',
+           r'io/read': 'GET',
+           r'camera/(?P<serial>[^\s /]+)/experiment/message$': 'POST',
+           r'experiment/message$': 'POST',
+           r'multicam/synchronize': 'POST',
+           r'multicam/connect_camera/(?P<serial>[^\s /]+)': 'POST',
+           r'multicam/disconnect_camera/(?P<serial>[^\s /]+)': 'POST',
+           r'multicam/connect_all': 'POST',
+           r'multicam/disconnect_all': 'POST',
+           }
+
+    def __init__(self, host=None, api_key=None, port=None, ca_cert=None, api_version=1):
+        self._log = logging.getLogger('motifapi')
+
+        if port is None:
+            try:
+                port = int(os.environ.get('MOTIF_PORT', 6083))
+            except:
+                port = 6083
+        else:
+            port = int(port)
+
+        if host is None:
+            try:
+                host = os.environ['MOTIF_HOST']
+                self._log.debug('took host from environment')
+            except KeyError:
+                pass
+            if not host:
+                host = '127.0.0.1'
+
+        if api_key is None:
+            try:
+                api_key = os.environ['MOTIF_API_KEY']
+                self._log.debug('took api_key from environment')
+            except KeyError:
+                pass
+            if not api_key:
+                try:
+                    api_key = subprocess.check_output(['recnode-apikey']).strip()
+                    self._log.debug('took api-key from recnode-apikey subprocess')
+                except OSError:
+                    pass
+
+        if not api_key:
+            raise ValueError('API key must be specified')
+
+        if ca_cert is None:
+            ca_cert = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'server.crt')
+
+        if not os.path.exists(ca_cert):
+            raise ValueError('could not find certificate: %s' % ca_cert)
+
+        self._prefix = 'api/%d/' % api_version
+
+        client_cert_key = None
+        client_cert_pem = None  # file path
+        handler = HTTPSClientAuthHandler(
+            key=client_cert_key,
+            cert=client_cert_pem,
+            ca_certs=ca_cert)
+        self._http = urllib.request.build_opener(handler)
+
+        self._host = host
+        self._api_key = api_key
+        self._port = port
+
+    def _build_request(self, endpoint, data=None, method='GET'):
+        if endpoint[0] == '/':
+            endpoint = endpoint[1:]
+
+        if endpoint != 'version':
+            endpoint = self._prefix + endpoint
+
+        if data is not None:
+            try:
+                data = json.dumps(data).encode("utf-8")
+            except TypeError:
+                raise ValueError('Arguments must be JSON serializable (they were %r)' % (data,))
+
+        url = 'https://%s:%s/%s' % (self._host, self._port, endpoint)
+
+        req = MethodRequest(url, data, method=method)
+        req.add_header('X-Api-Key', self._api_key)
+        req.add_header('Content-Type', 'application/json')
+
+        self._log.debug('%s %s (%d bytes %s)' % (method,
+                                                 url,
+                                                 0 if data is None else len(data),
+                                                 type(data)))
+
+        return req
+
+    def _call(self, req):
+        try:
+            resp = self._http.open(req)
+            data = resp.read()
+            resp.close()
+            return data
+        except urllib.error.HTTPError as e:
+            try:
+                raw = e.read()
+                err = json.loads(raw.decode('utf-8'))
+                exc = MotifApiError(err['error'], err['status_code'])
+            except Exception:
+                raise ValueError('unknown API error')
+            raise exc
+        except urllib.error.URLError:
+            raise MotifError('motif not running or reachable')
+
+    def call(self, endpoint, method=None, data=None, **kwargs):
+        meth = ep = None
+        for _ep, _meth in self.API.items():
+            if re.match(_ep, endpoint):
+                meth = _meth
+                ep = _ep
+                break
+
+        if meth is None:
+            raise ValueError("unknown endpoint '%s' (are you missing/adding '/')" % endpoint)
+
+        req = self._build_request(endpoint,
+                                  data=kwargs or None,
+                                  method=meth)
+        out = self._call(req)
+        if out:
+            try:
+                return json.loads(out.decode('utf-8'))
+            except ValueError as e:
+                raise ValueError('Invalid JSON response: %s' % e.message)
+        else:
+            return {}
+
+    def is_recording(self, serial):
+        r = self.call('camera/%s' % serial)
+        if r['camera_info'].get('filename'):
+            return True
+        elif r['camera_info'].get('status') == 'pending':
+            return True
+        return False
+
+    def is_copying(self, serial):
+        r = self.call('camera/%s' % serial)
+        return r['playback_info']['status'] == 'copying'
+
+    def is_exporting(self, serial):
+        r = self.call('camera/%s' % serial)
+        status = r['playback_info']['status']
+        return status.startswith('export') and ('finished' not in status)
+
+    def send_message(self, message, serial=None):
+        if serial:
+            return self.call('camera/%s/experiment/message' % serial, message=message)
+        return self.call('experiment/message', message=message)
+
+    def get_stream(self, serial=None, stream_type=STREAM_TYPE_IMAGE, force_host=None):
+        from .stream import ImageStreamer, StateStreamer
+
+        if serial is None:
+            try:
+                # get the first camera
+                for c in self.call('cameras').get('cameras', []):
+                    serial = c['serial']
+                    break
+            except (urllib.error.URLError, MotifApiError):
+                raise MotifError('motif not running or reachable')
+
+        if serial is None:
+            raise MotifError('no cameras connected or running')
+
+        def _get_host_port(_stream_name):
+            _stat = self.call('camera/%s' % serial)
+            _host = _stat['camera_info']['stream'][_stream_name]['host']
+            _port = int(_stat['camera_info']['stream'][_stream_name]['port'])
+
+            if force_host is not None:
+                _host = force_host
+            elif (_host == '0.0.0.0') and (self._host != '0.0.0.0'):
+                _host = self._host
+
+            return _host, _port
+
+        if stream_type in (MotifApi.STREAM_TYPE_IMAGE, MotifApi.STREAM_TYPE_STATE):
+            try:
+                if stream_type == MotifApi.STREAM_TYPE_IMAGE:
+                    host, port = _get_host_port('image')
+                    return ImageStreamer(host, port)
+                else:
+                    host, port = _get_host_port('state')
+                    return StateStreamer(host, port)
+            except (urllib.error.URLError, MotifApiError):
+                raise MotifError('camera with serial %s not found or running' % serial)
+            except KeyError:
+                raise MotifError('realtime stream not enabled on camera')
+        else:
+            raise ValueError('unknown stream type')
+
+
+

motifapi-0.2.0/motifapi/schedule.py

@@ -0,0 +1,2 @@
+def datetime_to_cron(dt):
+    return '%d %d %d %d %d ?' % (dt.second, dt.minute, dt.hour, dt.day, dt.month)

motifapi-0.2.0/motifapi/server.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDGzCCAgOgAwIBAgIJAOjl0YTg0v1KMA0GCSqGSIb3DQEBCwUAMCQxCzAJBgNV
+BAYTAkFVMRUwEwYDVQQKDAxsb29wYmlvIGdtYmgwHhcNMTcxMDI4MTM1MDQzWhcN
+MjcxMDI2MTM1MDQzWjAkMQswCQYDVQQGEwJBVTEVMBMGA1UECgwMbG9vcGJpbyBn
+bWJoMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuBDT2rvtlC3z330H
+p5Zl1ilA4gvSM7Zlvpwt+kQETb7rCS49+oPaAa89tDEbwrk3rDzxnGG2+YL73h07
+YpTlJk/7WkciGgR7K83CwEiSVUUZsmxfXH/t0V5km4d+Ul7lRWoRI9BcnWu9Mvwr
+572NQJf0cWs0FJu5qYmY0nI7+EjN/qB7OFYYTYSJZQYwoRZzdGFNWCiTkxTBOQc/
+1dyPzmtMbRQONYryMqMHKLUeG4aEYwCsm2XNSTHn7JCxc/dHc8BnBMkAOqNp+XP+
+n4blQ6e4C4e4MWEk/yrv0p8u0m6NIoBvI/nZDhgvdIQTFrj+eIOSCPCtxzAQEnDd
+f9FEiQIDAQABo1AwTjAdBgNVHQ4EFgQU1kn+nli9zgqxumR2zgMFS5WljhgwHwYD
+VR0jBBgwFoAU1kn+nli9zgqxumR2zgMFS5WljhgwDAYDVR0TBAUwAwEB/zANBgkq
+hkiG9w0BAQsFAAOCAQEAdr76akM25qXFURAgWszcjJAaaimp9RRUaXwO5UxVegCR
+x1FCZyfm/TIWDgfKjlPFiPgCS3WBL0gLWummZhJ5YFwd0lI34H9KXl9JgKidyNrH
+aDHBj4On0eu84f6nMxqFPFDDscJLlMvmfBYRFq/7QryU/LuWeSWhn/oQs4vakoRH
+aJxzot67YvH9N8qxX/6Nc2TeEAz4X5AyEYe46mAc99rRROWjnYD39dfmWzv/5Cge
+42YwNxtba8njGyPnCb+/kKerc+jXNBoopBGECBsMyGx3p5cVCnz//NhgvWmvquEU
+msK/FxvWDIB5Y/KEfeaghB4D4hgy3IF9Z6T9ZBsLsg==
+-----END CERTIFICATE-----

motifapi-0.2.0/motifapi/stream.py

@@ -0,0 +1,100 @@
+import sys
+import json
+import threading
+import logging
+
+import zmq
+import numpy as np
+
+
+if sys.version_info > (3,):
+    buffer = memoryview
+
+
+LOG = logging.getLogger('motifapi.stream')
+
+
+def recv_array(socket, flags=0, copy=True, track=False):
+    md = socket.recv_json(flags=flags)
+    msg = socket.recv(flags=flags, copy=copy, track=track)
+    # note: I don't actually think this is necessary, in PY2 this is type<str> and np.frombuffer(str) does
+    # the right thing, in PY3 this is type<bytes> which supports buffer protocol
+    buf = buffer(msg)
+    A = np.frombuffer(buf, dtype=md.pop('dtype'))
+    return A.reshape(md.pop('shape')), md
+
+
+class ImageStreamer(object):
+
+    stream = None
+
+    def __init__(self, host, port):
+        ctx = zmq.Context()
+        address = "tcp://%s:%d" % (host, port)
+        LOG.debug('image stream connecting to: %s' % address)
+        sock = ctx.socket(zmq.PULL)
+        sock.connect(address)
+        self.stream = sock
+
+    def get_next_image(self, block=True, copy=True):
+        while True:
+            while self.stream.poll(0, zmq.POLLIN):
+                return recv_array(self.stream, copy=copy)
+            if not block:
+                return None, None
+
+
+class StateStreamer(object):
+
+    def __init__(self, host, port, channel='j'):
+        ctx = zmq.Context()
+        address = "tcp://%s:%d" % (host, port)
+        LOG.debug('state stream connecting to: %s' % address)
+        sock = ctx.socket(zmq.SUB)
+        sock.connect(address)
+        sock.setsockopt(zmq.LINGER, 0)
+        if sys.version_info > (3,):
+            sock.setsockopt_string(zmq.SUBSCRIBE, channel)
+        else:
+            sock.setsockopt(zmq.SUBSCRIBE, channel)
+        self.stream = sock
+
+    def get_next_state(self, block=True):
+        while True:
+            _, msg = self.stream.recv_multipart()
+            return json.loads(msg)
+
+
+class StateMirror(threading.Thread):
+
+    daemon = True
+
+    def __init__(self, streamer):
+        assert isinstance(streamer, StateStreamer)
+        threading.Thread.__init__(self)
+        self._lock = threading.Lock()
+        self._state = {}
+        self._streamer = streamer
+
+    @property
+    def is_recording(self):
+        return self._state.get('is_recording', False)
+
+    @property
+    def frame_number(self):
+        return self._state.get('frame_number', None)
+
+    @property
+    def frame_time(self):
+        return self._state.get('frame_time', None)
+
+    def get_state(self, **extra):
+        with self._lock:
+            extra.update(self._state)
+            return extra
+
+    def run(self):
+        while True:
+            _, msg = self._streamer.stream.recv_multipart()
+            with self._lock:
+                self._state.update(json.loads(msg))

motifapi-0.2.0/motifapi/util.py

@@ -0,0 +1,16 @@
+import os
+import json
+
+
+def get_experiment_metadata():
+    try:
+        mdf = os.environ['MOTIF_METADATA_JSON_PATH']
+        with open(mdf, 'r') as f:
+            return json.load(f)
+    except KeyError:
+        # not defined
+        pass
+    except Exception as exc:
+        print("Error parsing metadata file", exc)
+
+    return {}

motifapi-0.2.0/motifapi.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: motifapi
+Version: 0.2.0
+Summary: Python interface to Motif recording systems
+Author: John Stowers
+Author-email: john@loopbio.com
+License: BSD
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.6
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: license
+Dynamic: requires-python
+Dynamic: summary
+
+Python interface to Motif recording systems

motifapi-0.2.0/motifapi.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+MANIFEST.in
+README.md
+setup.py
+motifapi/__init__.py
+motifapi/api.py
+motifapi/schedule.py
+motifapi/server.crt
+motifapi/stream.py
+motifapi/util.py
+motifapi.egg-info/PKG-INFO
+motifapi.egg-info/SOURCES.txt
+motifapi.egg-info/dependency_links.txt
+motifapi.egg-info/top_level.txt

motifapi-0.2.0/motifapi.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

motifapi-0.2.0/motifapi.egg-info/top_level.txt

@@ -0,0 +1 @@
+motifapi

motifapi-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

motifapi-0.2.0/setup.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup(
+    name='motifapi',
+    license='BSD',
+    description='Python interface to Motif recording systems',
+    long_description='Python interface to Motif recording systems',
+    version='0.2.0',
+    author='John Stowers',
+    author_email='john@loopbio.com',
+    packages=['motifapi'],
+    python_requires='>=3.6',
+    classifiers=[
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3 :: Only',
+    ],
+    include_package_data=True,
+)
+