zou 0.19.15__py3-none-any.whl → 0.20.11__py3-none-any.whl

zou/app/stores/auth_tokens_store.py

@@ -9,6 +9,7 @@ try:
         host=config.KEY_VALUE_STORE["host"],
         port=config.KEY_VALUE_STORE["port"],
         db=config.AUTH_TOKEN_BLACKLIST_KV_INDEX,
+        password=config.KEY_VALUE_STORE["password"],
         decode_responses=True,
     )
     revoked_tokens_store.ping()
@@ -58,4 +59,4 @@ def is_revoked(jti):
     """
     Tell if a stored auth token is revoked or not.
     """
-    return get(jti) in [None, "true"]
+    return get(jti) == "true"

zou/app/stores/file_store.py

@@ -104,6 +104,12 @@ def get_local_picture_path(prefix, id):
     return path(pictures, make_key(prefix, id))
 
 
+def copy_picture(prefix, id, new_prefix, new_id):
+    key = make_key(prefix, id)
+    target = make_key(new_prefix, new_id)
+    return pictures.copy(key, target)
+
+
 def add_movie(prefix, id, path):
     key = make_key(prefix, id)
     with open(path, "rb") as fd:
@@ -139,6 +145,12 @@ def get_local_movie_path(prefix, id):
     return path(movies, make_key(prefix, id))
 
 
+def copy_movie(prefix, id, new_prefix, new_id):
+    key = make_key(prefix, id)
+    target = make_key(new_prefix, new_id)
+    return movies.copy(key, target)
+
+
 def add_file(prefix, id, path):
     key = make_key(prefix, id)
     with open(path, "rb") as fd:
@@ -172,3 +184,9 @@ def remove_file(prefix, id):
 
 def get_local_file_path(prefix, id):
     return path(files, make_key(prefix, id))
+
+
+def copy_file(prefix, id, new_prefix, new_id):
+    key = make_key(prefix, id)
+    target = make_key(new_prefix, new_id)
+    return files.copy(key, target)

zou/app/stores/publisher_store.py

@@ -3,11 +3,7 @@ import redis
 from flask_socketio import SocketIO
 
 from zou.app import config
-
-host = config.KEY_VALUE_STORE["host"]
-port = config.KEY_VALUE_STORE["port"]
-redis_db = config.KV_EVENTS_DB_INDEX
-redis_url = "redis://%s:%s/%s" % (host, port, redis_db)
+from zou.app.utils.redis import get_redis_url
 
 socketio = None
 
@@ -27,11 +23,15 @@ def init():
 
     try:
         publisher_store = redis.StrictRedis(
-            host=host, port=port, db=redis_db, decode_responses=True
+            host=config.KEY_VALUE_STORE["host"],
+            port=config.KEY_VALUE_STORE["port"],
+            db=config.KV_EVENTS_DB_INDEX,
+            password=config.KEY_VALUE_STORE["password"],
+            decode_responses=True,
         )
         publisher_store.get("test")
         socketio = SocketIO(
-            message_queue=redis_url,
+            message_queue=get_redis_url(config.KV_EVENTS_DB_INDEX),
             cors_allowed_origins=[],
             cors_credentials=False,
         )

zou/app/stores/queue_store.py

@@ -11,6 +11,7 @@ try:
             host=config.KEY_VALUE_STORE["host"],
             port=config.KEY_VALUE_STORE["port"],
             db=config.KV_JOB_DB_INDEX,
+            password=config.KEY_VALUE_STORE["password"],
             decode_responses=True,
         )
         queue_store.get("test")

zou/app/swagger.py

@@ -14,6 +14,7 @@ swagger_config = {
     "static_url_path": "/docs",
     "swagger_ui": True,
     "specs_route": "/apidocs/",
+    "openapi": "3.0.2",
 }
 
 
@@ -30,26 +31,18 @@ An easy to use Python client to access this API is available:
 
 <p>Before you can use any of the endpoints outline below,
 you will have to get a JWT token to authorize your requests.
+</p>
 
-You can get a authorization token using a (form-encoded) POST request to ```/auth/login```.
-With curl this would look something like ```curl -X POST <server_address>/auth/login -d "email=<youremail>&password=<yourpassword>```.
+<p>
+You will find the information to retrieve it in the 
+<a href="#tag/Authentication">Zou documentation</a>.
+</p>
 
-The response is a JSON object, specifically you'll need to provide the ```access_token``` for your future requests.
-
-Here is a complete authentication process as an example (again using curl):
-```
-$ curl -X POST <server_address>/api/auth/login -d "email=<youremail>&password=<yourpassword>"'
-{{"login": true", "access_token": "eyJ0e...", ...}}
-$ jwt=eyJ0e...  # Store the access token for easier use
-$ curl -H "Authorization: Bearer $jwt" <server_address>/api/data/projects
-[{{...}},
-{{...}}]
-```
 [OpenAPI definition](/openapi.json)
 """
 
 swagger_template = {
-    "openapi": "3.1",
+    "openapi": "3.0.2",
     "info": {
         "title": "Kitsu API",
         "description": description,
@@ -70,12 +63,14 @@ swagger_template = {
     "host": "localhost:8080",
     "basePath": "/api",
     "schemes": ["http", "https"],
-    "securityDefinitions": {
-        "JWT Authorization": {
-            "name": "Authorization",
-            "in": "header",
-            "type": "apiKey",
-            "description": "Format in header: **Authorization: Bearer {token}**. \n\n Value example: Bearer xxxxx.yyyyy.zzzzz",
+    "components": {
+        "securitySchemes": {
+            "JWT Authorization": {
+                "name": "Authorization",
+                "in": "header",
+                "type": "apiKey",
+                "description": "Format in header: **Authorization: Bearer {token}**. \n\n Value example: Bearer xxxxx.yyyyy.zzzzz",
+            }
         }
     },
     "security": [{"JWT Authorization": []}],
@@ -345,6 +340,10 @@ swagger_template = {
                     "type": "string",
                     "description": "Color of department",
                 },
+                "archived": {
+                    "type": "boolean",
+                    "description": "True if the department is archived else False",
+                },
             },
         },
         "DesktopLoginLog": {
@@ -1189,6 +1188,23 @@ swagger_template = {
                 },
             },
         },
+        "Studio": {
+            "type": "object",
+            "properties": {
+                "name": {
+                    "type": "string",
+                    "description": "Name of studio",
+                },
+                "color": {
+                    "type": "string",
+                    "description": "Color of studio",
+                },
+                "archived": {
+                    "type": "boolean",
+                    "description": "True if the studio is archived else False",
+                },
+            },
+        },
         "Task": {
             "type": "object",
             "properties": {

zou/app/utils/cache.py

@@ -17,6 +17,7 @@ try:
         host=config.KEY_VALUE_STORE["host"],
         port=config.KEY_VALUE_STORE["port"],
         db=config.MEMOIZE_DB_INDEX,
+        password=config.KEY_VALUE_STORE["password"],
         decode_responses=True,
     )
     redis_cache.get("test")
@@ -26,6 +27,7 @@ try:
             "CACHE_REDIS_HOST": config.KEY_VALUE_STORE["host"],
             "CACHE_REDIS_PORT": config.KEY_VALUE_STORE["port"],
             "CACHE_REDIS_DB": config.MEMOIZE_DB_INDEX,
+            "CACHE_REDIS_PASSWORD": config.KEY_VALUE_STORE["password"],
         }
     )
 

zou/app/utils/commands.py

@@ -3,11 +3,13 @@
 import os
 import datetime
 import tempfile
+import sys
+import shutil
 
 
 from ldap3 import Server, Connection, ALL, NTLM, SIMPLE
 from zou.app.utils import thumbnail as thumbnail_utils, auth
-from zou.app.stores import auth_tokens_store, file_store
+from zou.app.stores import auth_tokens_store, file_store, queue_store
 from zou.app.services import (
     assets_service,
     backup_service,
@@ -23,6 +25,8 @@ from zou.app.services import (
     tasks_service,
 )
 from zou.app.models.person import Person
+from zou.app.models.preview_file import PreviewFile
+from zou.app.models.task import Task
 from sqlalchemy.sql.expression import not_
 
 from zou.app.services.exception import (
@@ -390,7 +394,9 @@ def sync_with_ldap_server():
             )
             .all()
         ):
-            persons_service.update_person(person.id, {"active": False})
+            persons_service.update_person(
+                person.id, {"active": False}, bypass_protected_accounts=True
+            )
             print(
                 "User %s disabled (not found in LDAP)." % person.desktop_login
             )
@@ -414,6 +420,7 @@ def sync_with_ldap_server():
                         "desktop_login": user["desktop_login"],
                         "ldap_uid": user["ldap_uid"],
                     },
+                    bypass_protected_accounts=True,
                 )
                 print(f"User {user['desktop_login']} updated.")
             except IsUserLimitReachedException:
@@ -466,7 +473,9 @@ def sync_with_ldap_server():
         )
         file_store.add_picture("thumbnails", person["id"], thumbnail_png_path)
         os.remove(thumbnail_png_path)
-        persons_service.update_person(person["id"], {"has_avatar": True})
+        persons_service.update_person(
+            person["id"], {"has_avatar": True}, bypass_protected_accounts=True
+        )
 
     ldap_users = get_ldap_users()
     update_person_list_with_ldap_users(ldap_users)
@@ -529,7 +538,7 @@ def run_sync_file_change_daemon(
 
 
 def import_last_changes_from_another_instance(
-    source, login, password, minutes=0, page_size=300
+    source, login, password, minutes=0, limit=300
 ):
     """
     Retrieve and save all the data related to most recent events from another
@@ -538,12 +547,12 @@ def import_last_changes_from_another_instance(
     with app.app_context():
         sync_service.init(source, login, password)
         print("Last events syncing started.")
-        sync_service.run_last_events_sync(minutes=minutes, page_size=300)
+        sync_service.run_last_events_sync(minutes=minutes, limit=300)
         print("Last events syncing ended.")
 
 
 def import_last_file_changes_from_another_instance(
-    source, login, password, minutes=20, page_size=50, force=False
+    source, login, password, minutes=20, limit=50, force=False
 ):
     """
     Retrieve and save all the data related most to recent file events
@@ -553,7 +562,7 @@ def import_last_file_changes_from_another_instance(
     with app.app_context():
         sync_service.init(source, login, password)
         print("Last files syncing started.")
-        sync_service.run_last_events_files(minutes=minutes, page_size=50)
+        sync_service.run_last_events_files(minutes=minutes, limit=50)
         print("Last files syncing ended.")
 
 
@@ -717,3 +726,91 @@ def create_bot(
             is_bot=True,
         )
         print(bot["access_token"])
+
+
+def renormalize_movie_preview_files(
+    preview_file_id=None, project_id=None, all_broken=None, all_processing=None
+):
+    with app.app_context():
+        if preview_file_id is None and not all_broken and not all_processing:
+            print("You must specify at least one option.")
+            sys.exit(1)
+        else:
+            query = PreviewFile.query.filter(
+                PreviewFile.extension == "mp4"
+            ).order_by(PreviewFile.created_at.asc())
+
+            if preview_file_id is not None:
+                query = query.filter(PreviewFile.id == preview_file_id)
+
+            if project_id is not None:
+                query = query.join(Task).filter(
+                    PreviewFile.project_id == project_id
+                )
+
+            if all_broken and all_processing:
+                query = query.filter(
+                    PreviewFile.status.in_(("broken", "processing"))
+                )
+            elif all_broken:
+                query = query.filter(PreviewFile.status == "broken")
+            elif all_processing:
+                query = query.filter(PreviewFile.status == "processing")
+
+            preview_files = query.all()
+            len_preview_files = len(preview_files)
+            if len_preview_files == 0:
+                print("No preview files found.")
+                sys.exit(1)
+            else:
+                for i, preview_file in enumerate(preview_files):
+                    try:
+                        preview_file_id = str(preview_file.id)
+                        print(
+                            f"Renormalizing preview file {preview_file_id} ({i+1}/{len_preview_files})."
+                        )
+                        extension = preview_file.extension
+                        uploaded_movie_path = os.path.join(
+                            config.TMP_DIR,
+                            f"{preview_file_id}.{extension}.tmp",
+                        )
+                        try:
+                            if config.FS_BACKEND == "local":
+                                shutil.copyfile(
+                                    file_store.get_local_movie_path(
+                                        "source", preview_file_id
+                                    ),
+                                    uploaded_movie_path,
+                                )
+                            else:
+                                sync_service.download_file(
+                                    uploaded_movie_path,
+                                    "source",
+                                    file_store.open_movie,
+                                    str(preview_file_id),
+                                )
+                        except:
+                            pass
+                        if config.ENABLE_JOB_QUEUE:
+                            queue_store.job_queue.enqueue(
+                                preview_files_service.prepare_and_store_movie,
+                                args=(
+                                    preview_file_id,
+                                    uploaded_movie_path,
+                                    True,
+                                    False,
+                                ),
+                                job_timeout=int(config.JOB_QUEUE_TIMEOUT),
+                            )
+                        else:
+                            preview_files_service.prepare_and_store_movie(
+                                preview_file_id,
+                                uploaded_movie_path,
+                                normalize=True,
+                                add_source_to_file_store=False,
+                            )
+                    except Exception as e:
+                        print(
+                            f"Renormalization of preview file {preview_file_id} failed: {e}"
+                        )
+                        continue

zou/app/utils/csv_utils.py

@@ -1,9 +1,6 @@
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
 import csv
 
+from io import StringIO
 from flask import make_response
 from slugify import slugify
 

zou/app/utils/date_helpers.py

@@ -1,17 +1,21 @@
 import isoweek
+import datetime
 
 from babel.dates import format_datetime
-from datetime import date, datetime, timedelta
 from dateutil import relativedelta
 from zou.app.services.exception import WrongDateFormatException
 
 
 def get_now():
-    return get_string_with_timezone_from_date(datetime.utcnow(), "UTC")
+    return get_string_with_timezone_from_date(get_utc_now_datetime(), "UTC")
+
+
+def get_utc_now_datetime():
+    return datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)
 
 
 def get_date_from_now(nb_days):
-    return date.today() - timedelta(days=nb_days)
+    return datetime.date.today() - datetime.timedelta(days=nb_days)
 
 
 def get_date_diff(date_a, date_b):
@@ -29,7 +33,7 @@ def get_date_string_with_timezone(date_string, timezone):
     """
     Apply given timezone to given date and return it as a string.
     """
-    date_obj = datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S")
+    date_obj = datetime.datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S")
     return format_datetime(date_obj, "yyyy-MM-ddTHH:mm:ss", tzinfo=timezone)
 
 
@@ -51,14 +55,16 @@ def get_today_string_with_timezone(timezone):
     """
     Get today date in string format with timezone applied.
     """
-    return get_simple_string_with_timezone_from_date(date.today(), timezone)
+    return get_simple_string_with_timezone_from_date(
+        datetime.date.today(), timezone
+    )
 
 
 def get_date_from_string(date_str):
     """
     Parse a date string and returns a date object.
     """
-    return datetime.strptime(date_str, "%Y-%m-%d")
+    return datetime.datetime.strptime(date_str, "%Y-%m-%d")
 
 
 def get_datetime_from_string(date_str, milliseconds=False):
@@ -66,9 +72,9 @@ def get_datetime_from_string(date_str, milliseconds=False):
     Parse a datetime string and returns a datetime object.
     """
     if milliseconds:
-        return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%f")
+        return datetime.datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%f")
     else:
-        return datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
+        return datetime.datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
 
 
 def get_year_interval(year):
@@ -76,10 +82,10 @@ def get_year_interval(year):
     Get a tuple containing start date and end date for given year.
     """
     year = int(year)
-    if year > datetime.utcnow().year or year < 2010:
+    if year > get_utc_now_datetime().year or year < 2010:
         raise WrongDateFormatException
 
-    start = datetime(year, 1, 1)
+    start = datetime.datetime(year, 1, 1)
     end = start + relativedelta.relativedelta(years=1)
     return start, end
 
@@ -90,10 +96,15 @@ def get_month_interval(year, month):
     """
     year = int(year)
     month = int(month)
-    if year > datetime.utcnow().year or year < 2010 or month < 1 or month > 12:
+    if (
+        year > get_utc_now_datetime().year
+        or year < 2010
+        or month < 1
+        or month > 12
+    ):
         raise WrongDateFormatException
 
-    start = datetime(year, month, 1)
+    start = datetime.datetime(year, month, 1)
     end = start + relativedelta.relativedelta(months=1)
     return start, end
 
@@ -104,7 +115,12 @@ def get_week_interval(year, week):
     """
     year = int(year)
     week = int(week)
-    if year > datetime.utcnow().year or year < 2010 or week < 1 or week > 52:
+    if (
+        year > get_utc_now_datetime().year
+        or year < 2010
+        or week < 1
+        or week > 52
+    ):
         raise WrongDateFormatException
     start = isoweek.Week(year, week).monday()
     end = start + relativedelta.relativedelta(days=7)
@@ -119,7 +135,7 @@ def get_day_interval(year, month, day):
     month = int(month)
     day = int(day)
     if (
-        year > datetime.utcnow().year
+        year > get_utc_now_datetime().year
         or year < 2010
         or month < 1
         or month > 12
@@ -127,7 +143,7 @@ def get_day_interval(year, month, day):
         or day > 31
     ):
         raise WrongDateFormatException
-    start = datetime(year, month, day)
+    start = datetime.datetime(year, month, day)
     end = start + relativedelta.relativedelta(days=1)
     return start, end
 
@@ -147,14 +163,14 @@ def get_business_days(start, end):
     Returns the number of business days between two dates.
     """
     daygenerator = (
-        start + timedelta(x + 1) for x in range((end - start).days)
+        start + datetime.timedelta(x + 1) for x in range((end - start).days)
     )
     return sum(1 for day in daygenerator if day.weekday() < 5)
 
 
 def add_business_days_to_date(date, nb_days):
     while nb_days > 0:
-        date += timedelta(days=1)
+        date += datetime.timedelta(days=1)
         if date.weekday() < 5:
             nb_days -= 1
     return date

zou/app/utils/dbhelpers.py

@@ -1,4 +1,4 @@
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, inspect
 from sqlalchemy_utils import database_exists, create_database
 from sqlalchemy.engine.url import URL
 from sqlalchemy.orm import close_all_sessions
@@ -39,3 +39,16 @@ def drop_all():
     db.session.flush()
     close_all_sessions()
     return db.drop_all()
+
+
+def is_init():
+    """
+    Check if database is initialized.
+    """
+    from zou.app import db
+    from zou.app.models.project_status import ProjectStatus
+
+    return (
+        inspect(db.engine).has_table("person")
+        and db.session.query(ProjectStatus).count() == 2
+    )

zou/app/utils/emails.py

@@ -12,9 +12,9 @@ def send_email(subject, html, recipient_email, body=None):
     """
     if body is None:
         body = strip_html_tags(html)
-    if app.config["MAIL_DEBUG"]:
+    if app.config["MAIL_DEBUG_BODY"]:
         print(body)
-    elif app.config["MAIL_ENABLED"]:
+    if app.config["MAIL_ENABLED"]:
         with app.app_context():
             try:
                 mail_default_sender = app.config["MAIL_DEFAULT_SENDER"]

zou/app/utils/fido.py

@@ -0,0 +1,22 @@
+import fido2.features
+
+from fido2.webauthn import (
+    PublicKeyCredentialRpEntity,
+)
+from fido2.server import Fido2Server
+from urllib.parse import urlparse
+
+from zou.app import config
+
+fido2.features.webauthn_json_mapping.enabled = True
+
+
+def get_fido_server():
+    return Fido2Server(
+        PublicKeyCredentialRpEntity(
+            name="Kitsu", id=urlparse(f"https://{config.DOMAIN_NAME}").hostname
+        ),
+        verify_origin=(
+            None if config.DOMAIN_NAME != "localhost:8080" else lambda a: True
+        ),
+    )

zou/app/utils/query.py

@@ -1,8 +1,11 @@
 import math
+import orjson as json
+import sqlalchemy.orm as orm
 
 from zou.app import app
 from zou.app.utils import fields, string
 from sqlalchemy import func
+from sqlalchemy.inspection import inspect
 
 
 def get_query_criterions_from_request(request):
@@ -21,12 +24,57 @@ def apply_criterions_to_db_query(model, db_query, criterions):
     """
     Apply criterions given in HTTP request to the sqlachemy db query object.
     """
-    if "name" in criterions and hasattr(model, "name"):
-        value = criterions["name"]
-        db_query = db_query.filter(model.name.ilike(value))
-        del criterions["name"]
 
-    return db_query.filter_by(**criterions)
+    many_join_filter = []
+    in_filter = []
+    name_filter = []
+    filters = {}
+
+    column_names = inspect(model).all_orm_descriptors.keys()
+    for key, value in criterions.items():
+        if key not in ["page", "relations"] and key in column_names:
+            field_key = getattr(model, key)
+
+            is_many_to_many_field = hasattr(
+                field_key, "property"
+            ) and isinstance(
+                field_key.property, orm.properties.RelationshipProperty
+            )
+            value_is_list = (
+                hasattr(value, "__len__")
+                and len(value) > 0
+                and value[0] == "["
+            )
+
+            if key == "name" and field_key is not None:
+                name_filter.append(value)
+
+            elif is_many_to_many_field:
+                many_join_filter.append((key, value))
+
+            elif value_is_list:
+                value_array = json.loads(value)
+                in_filter.append(
+                    field_key.in_(
+                        [cast_value(value, field_key) for value in value_array]
+                    )
+                )
+            else:
+                filters[key] = cast_value(value, field_key)
+
+        if filters:
+            db_query = db_query.filter_by(**filters)
+
+        for value in name_filter:
+            db_query = db_query.filter(model.name.ilike(value))
+
+        for id_filter in in_filter:
+            db_query = db_query.filter(id_filter)
+
+        for key, value in many_join_filter:
+            db_query = db_query.filter(getattr(model, key).any(id=value))
+
+    return db_query
 
 
 def get_paginated_results(query, page, limit=None, relations=False):
@@ -79,7 +127,7 @@ def get_cursor_results(
     total = query.count()
     query = (
         query.filter(model.created_at > cursor_created_at)
-        .order_by(model.created_at)
+        .order_by(model.created_at, model.updated_at, model.id)
         .limit(limit)
     )
     models = fields.serialize_models(