dsw-tdk 3.13.0__py3-none-any.whl → 4.27.0__py3-none-any.whl

dsw/tdk/core.py

@@ -0,0 +1,565 @@
+import asyncio
+import datetime
+import io
+import json
+import logging
+import pathlib
+import shutil
+import tempfile
+import zipfile
+
+import watchfiles
+
+from . import consts
+from .api_client import WizardAPIClient, WizardCommunicationError
+from .model import Template, TemplateFile, TemplateFileType, TemplateProject
+from .utils import UUIDGen
+from .validation import TemplateValidator, ValidationError
+
+
+ChangeItem = tuple[watchfiles.Change, pathlib.Path]
+
+
+def _change(item: ChangeItem, root: pathlib.Path) -> str:
+    return f'{item[0].name.upper()}[{item[1].relative_to(root).as_posix()}]'
+
+
+class TDKProcessingError(RuntimeError):
+
+    def __init__(self, message: str, hint: str):
+        self.message = message
+        self.hint = hint
+
+
+# pylint: disable-next=too-many-public-methods
+class TDKCore:
+
+    # pylint: disable-next=too-many-locals
+    def _check_metamodel_version(self):
+        hint = 'Fix your metamodelVersion in template.json and/or visit docs'
+        if self.remote_metamodel_version is None:
+            self.logger.warning('Remote metamodel version is unknown, you should use '
+                                'a matching version of TDK to avoid issues')
+            return
+        mm_ver = str(self.safe_template.metamodel_version)
+        try:
+            if '.' not in mm_ver:
+                mm_ver = f'{mm_ver}.0'
+            mm_major, mm_minor = map(int, mm_ver.split('.'))
+        except ValueError as e:
+            raise TDKProcessingError(f'Invalid metamodel version format: {mm_ver}', hint) from e
+        mmr_ver = self.remote_metamodel_version
+        try:
+            if '.' not in mmr_ver:
+                mmr_ver = f'{mmr_ver}.0'
+            mmr_major, mmr_minor = map(int, mmr_ver.split('.'))
+        except ValueError as e:
+            raise TDKProcessingError(f'Invalid remote metamodel version format: {mmr_ver}',
+                                     'Check if connecting to correct API with matching version '
+                                     'as you have in TDK.') from e
+        if (mm_major, mm_minor) == (mmr_major, mmr_minor):
+            self.logger.debug('Metamodel version %s matches remote version %s',
+                              mmr_ver, mmr_ver)
+        elif mm_major == mmr_major and mm_minor < mmr_minor:
+            self.logger.warning('Local metamodel version %s is older than remote version %s, '
+                                'but still compatible', mm_ver, mmr_ver)
+        else:
+            raise TDKProcessingError(
+                f'Unsupported metamodel version: local {mm_ver}, remote {mmr_ver}',
+                hint,
+            )
+
+    def __init__(self, template: Template | None = None, project: TemplateProject | None = None,
+                 client: WizardAPIClient | None = None, logger: logging.Logger | None = None):
+        self.template = template
+        self.project = project
+        self.client = client
+        self.remote_version: str = 'unknown~??????'
+        self.remote_metamodel_version: str | None = 'unknown'
+        self.logger = logger or logging.getLogger()
+        self.loop = asyncio.get_event_loop()
+        self.changes_processor = ChangesProcessor(self)
+        self.remote_id = 'unknown'
+
+    async def close(self):
+        await self.safe_client.close()
+
+    @property
+    def safe_template(self) -> Template:
+        if self.template is None:
+            raise RuntimeError('No template is loaded')
+        return self.template
+
+    @property
+    def safe_project(self) -> TemplateProject:
+        if self.project is None:
+            raise RuntimeError('No template is loaded')
+        return self.project
+
+    @property
+    def safe_client(self) -> WizardAPIClient:
+        if self.client is None:
+            raise RuntimeError('No DSW API client specified')
+        return self.client
+
+    async def init_client(self, api_url: str, api_key: str):
+        self.logger.info('Connecting to %s', api_url)
+        self.client = WizardAPIClient(api_url=api_url, api_key=api_key)
+        self.remote_version, self.remote_metamodel_version = await self.client.get_api_version()
+        user = await self.client.get_current_user()
+        self.logger.info('Successfully authenticated as %s %s (%s)',
+                         user['firstName'], user['lastName'], user['email'])
+        self.logger.debug('Connected to API version %s', self.remote_version)
+
+    def prepare_local(self, template_dir):
+        self.logger.debug('Preparing local template project')
+        self.project = TemplateProject(template_dir=template_dir, logger=self.logger)
+
+    def load_local(self, template_dir):
+        self.prepare_local(template_dir=template_dir)
+        self.logger.info('Loading local template project')
+        self.safe_project.load()
+
+    async def load_remote(self, template_id: str):
+        self.logger.info('Retrieving template draft %s', template_id)
+        self.template = await self.safe_client.get_template_draft(remote_id=template_id)
+        self.logger.debug('Retrieving template draft files')
+        files = await self.safe_client.get_template_draft_files(remote_id=template_id)
+        self.logger.info('Retrieved %s file(s)', len(files))
+        for file in files:
+            self.safe_template.files[file.filename.as_posix()] = file
+        self.logger.debug('Retrieving template draft assets')
+        assets = await self.safe_client.get_template_draft_assets(remote_id=template_id)
+        self.logger.info('Retrieved %s asset(s)', len(assets))
+        for asset in assets:
+            self.safe_template.files[asset.filename.as_posix()] = asset
+
+    async def download_bundle(self, template_id: str) -> bytes:
+        self.logger.info('Retrieving template %s bundle', template_id)
+        return await self.safe_client.get_template_bundle(remote_id=template_id)
+
+    async def list_remote_templates(self) -> list[Template]:
+        self.logger.info('Listing remote document templates')
+        return await self.safe_client.get_templates()
+
+    async def list_remote_drafts(self) -> list[Template]:
+        self.logger.info('Listing remote document template drafts')
+        return await self.safe_client.get_drafts()
+
+    def verify(self) -> list[ValidationError]:
+        template = self.template or self.safe_project.template
+        if template is None:
+            raise RuntimeError('No template is loaded')
+        return TemplateValidator.collect_errors(template)
+
+    def store_local(self, force: bool):
+        if self.project is None:
+            raise RuntimeError('No template project is initialized')
+        self.project.template = self.safe_template
+        if self.project.template is None:
+            raise RuntimeError('No template is loaded in the project')
+        if len(self.project.safe_template.tdk_config.files) == 0:
+            self.project.safe_template.tdk_config.use_default_files()
+            self.logger.warning('Using default _tdk.files in template.json, you may want '
+                                'to change it to include relevant files')
+        self.logger.debug('Initiating storing local template project (force=%s)', force)
+        self.project.store(force=force)
+
+    async def store_remote(self, force: bool):
+        self.template = self.safe_project.template
+        self._check_metamodel_version()
+        org_id = await self.safe_client.get_organization_id()
+        if org_id != self.safe_template.organization_id:
+            self.logger.warning('There is different organization ID set in the DSW instance'
+                                ' (local: %s, remote: %s)',
+                                self.safe_template.organization_id, org_id)
+        self.remote_id = self.safe_template.id_with_org(org_id)
+        template_exists = await self.safe_client.check_draft_exists(remote_id=self.remote_id)
+        if template_exists and force:
+            self.logger.warning('Deleting existing remote document template draft (forced)')
+            result = await self.safe_client.delete_template_draft(remote_id=self.remote_id)
+            if not result:
+                self.logger.error('Could not delete document template draft')
+            template_exists = not result
+
+        if template_exists:
+            self.logger.info('Updating existing remote document template draft')
+            await self.safe_client.update_template_draft(
+                template=self.safe_template,
+                remote_id=self.remote_id,
+            )
+            self.logger.debug('Retrieving remote assets')
+            remote_assets = await self.safe_client.get_template_draft_assets(
+                remote_id=self.remote_id,
+            )
+            self.logger.debug('Retrieving remote files')
+            remote_files = await self.safe_client.get_template_draft_files(
+                remote_id=self.remote_id,
+            )
+            await self.cleanup_remote_files(
+                remote_assets=remote_assets,
+                remote_files=remote_files,
+            )
+        else:
+            self.logger.info('Creating remote document template draft')
+            await self.safe_client.create_new_template_draft(
+                template=self.safe_template,
+                remote_id=self.remote_id,
+            )
+        await self.store_remote_files()
+
+    async def _update_template_file(self, remote_file: TemplateFile, local_file: TemplateFile,
+                                    project_update: bool = False):
+        try:
+            self.logger.debug('Updating existing remote %s %s (%s) started',
+                              remote_file.remote_type.value, remote_file.filename.as_posix(),
+                              remote_file.remote_id)
+            local_file.remote_id = remote_file.remote_id
+            if remote_file.remote_type == TemplateFileType.ASSET:
+                result = await self.safe_client.put_template_draft_asset_content(
+                    remote_id=self.remote_id,
+                    file=local_file,
+                )
+            else:
+                result = await self.safe_client.put_template_draft_file_content(
+                    remote_id=self.remote_id,
+                    file=local_file,
+                )
+            self.logger.debug('Updating existing remote %s %s (%s) finished: %s',
+                              remote_file.remote_type.value, remote_file.filename.as_posix(),
+                              remote_file.remote_id, 'ok' if result else 'failed')
+            if project_update and result:
+                self.safe_project.update_template_file(result)
+        except Exception as e1:
+            try:
+                self.logger.debug('Trying to delete/create due to: %s', str(e1))
+                await self._delete_template_file(file=remote_file)
+                await self._create_template_file(file=local_file, project_update=True)
+            except Exception as e2:
+                self.logger.error('Failed to update existing remote %s %s: %s',
+                                  remote_file.remote_type.value,
+                                  remote_file.filename.as_posix(), e2)
+
+    async def _delete_template_file(self, file: TemplateFile, project_update: bool = False):
+        try:
+            self.logger.debug('Deleting existing remote %s %s (%s) started',
+                              file.remote_type.value, file.filename.as_posix(),
+                              file.remote_id)
+            if file.remote_type == TemplateFileType.ASSET:
+                result = await self.safe_client.delete_template_draft_asset(
+                    remote_id=self.remote_id,
+                    asset_id=file.remote_id,
+                )
+            else:
+                result = await self.safe_client.delete_template_draft_file(
+                    remote_id=self.remote_id,
+                    file_id=file.remote_id,
+                )
+            self.logger.debug('Deleting existing remote %s %s (%s) finished: %s',
+                              file.remote_type.value,
+                              file.filename.as_posix(),
+                              file.remote_id, 'ok' if result else 'failed')
+            if project_update and result:
+                self.safe_project.remove_template_file(file.filename)
+        except Exception as e:
+            self.logger.error('Failed to delete existing remote %s %s: %s',
+                              file.remote_type.value, file.filename.as_posix(), e)
+
+    async def cleanup_remote_files(self, remote_assets: list[TemplateFile],
+                                   remote_files: list[TemplateFile]):
+        for local_file in self.safe_project.safe_template.files.values():
+            self.logger.debug('Cleaning up remote %s', local_file.filename.as_posix())
+            for remote_asset in remote_assets:
+                if remote_asset.filename == local_file.filename:
+                    await self._delete_template_file(file=remote_asset, project_update=False)
+            for remote_file in remote_files:
+                if remote_file.filename == local_file.filename:
+                    await self._delete_template_file(file=remote_file, project_update=False)
+
+    async def _create_template_file(self, file: TemplateFile, project_update: bool = False):
+        try:
+            self.logger.debug('Storing remote %s %s started',
+                              file.remote_type.value, file.filename.as_posix())
+            if file.remote_type == TemplateFileType.ASSET:
+                result = await self.safe_client.post_template_draft_asset(
+                    remote_id=self.remote_id,
+                    file=file,
+                )
+            else:
+                result = await self.safe_client.post_template_draft_file(
+                    remote_id=self.remote_id,
+                    file=file,
+                )
+            self.logger.debug('Storing remote %s %s finished: %s',
+                              file.remote_type.value, file.filename.as_posix(),
+                              result.remote_id)
+            if project_update and result is not None:
+                self.safe_project.update_template_file(result)
+        except Exception as e:
+            self.logger.error('Failed to store remote %s %s: %s',
+                              file.remote_type.value, file.filename.as_posix(), e)
+
+    async def store_remote_files(self):
+        if len(self.safe_project.safe_template.files) == 0:
+            self.logger.warning('No files to store, maybe you forgot to '
+                                'update _tdk.files patterns in template.json?')
+        for file in self.safe_project.safe_template.files.values():
+            file.remote_id = None
+            file.remote_type = TemplateFileType.FILE if file.is_text else TemplateFileType.ASSET
+            await self._create_template_file(file=file, project_update=True)
+
+    def create_package(self, output: pathlib.Path, force: bool):
+        if output.exists() and not force:
+            raise RuntimeError(f'File {output} already exists (not forced)')
+        self.logger.debug('Opening ZIP file for write: %s', output.as_posix())
+        with zipfile.ZipFile(output, mode='w', compression=zipfile.ZIP_DEFLATED) as pkg:
+            descriptor = self.safe_project.safe_template.serialize_remote()
+            files = []
+            assets = []
+            for file in self.safe_project.safe_template.files.values():
+                if file.is_text:
+                    self.logger.info('Adding template file %s', file.filename.as_posix())
+                    files.append({
+                        'uuid': str(UUIDGen.generate()),
+                        'content': file.content.decode(encoding=consts.DEFAULT_ENCODING),
+                        'fileName': str(file.filename.as_posix()),
+                    })
+                else:
+                    self.logger.info('Adding template asset %s',
+                                     file.filename.as_posix())
+                    assets.append({
+                        'uuid': str(UUIDGen.generate()),
+                        'contentType': file.content_type,
+                        'fileName': str(file.filename.as_posix()),
+                    })
+                    self.logger.debug('Packaging template asset %s',
+                                      file.filename.as_posix())
+                    pkg.writestr(f'template/assets/{file.filename.as_posix()}',
+                                 file.content)
+            descriptor['files'] = files
+            descriptor['assets'] = assets
+            if len(files) == 0 and len(assets) == 0:
+                self.logger.warning('No files or assets found in the template, maybe you forgot '
+                                    'to update _tdk.files patterns in template.json?')
+            timestamp = datetime.datetime.now(tz=datetime.UTC).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+            descriptor['createdAt'] = timestamp
+            descriptor['updatedAt'] = timestamp
+            self.logger.debug('Packaging template.json file')
+            pkg.writestr('template/template.json',
+                         data=json.dumps(descriptor, indent=4))
+        self.logger.debug('ZIP packaging done')
+
+    # pylint: disable=too-many-locals
+    def extract_package(self, zip_data: bytes, template_dir: pathlib.Path | None, force: bool):
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            io_zip = io.BytesIO(zip_data)
+            with zipfile.ZipFile(io_zip) as pkg:
+                pkg.extractall(tmp_dir)
+            del io_zip
+            tmp_root = pathlib.Path(tmp_dir) / 'template'
+            file = tmp_root / 'template.json'
+            assets_dir = tmp_root / 'assets'
+            self.logger.debug('Extracting template data')
+            if not file.exists():
+                raise RuntimeError('Malformed package: missing template.json file')
+            data = json.loads(file.read_text(encoding=consts.DEFAULT_ENCODING))
+            template = Template.load_local(data)
+            template.tdk_config.use_default_files()
+            self.logger.warning('Using default _tdk.files in template.json, you may want '
+                                'to change it to include relevant files')
+            self.logger.debug('Preparing template dir')
+            if template_dir is None:
+                template_dir = pathlib.Path.cwd() / template.id.replace(':', '_')
+            if template_dir.exists():
+                if not force:
+                    raise RuntimeError(f'Template directory already exists: '
+                                       f'{template_dir.as_posix()} (use force?)')
+                shutil.rmtree(template_dir, ignore_errors=True)
+            template_dir.mkdir(parents=True)
+            self.logger.debug('Extracting template.json from package')
+            local_template_json = template_dir / 'template.json'
+            local_template_json.write_text(
+                data=json.dumps(template.serialize_local_new(), indent=2),
+                encoding=consts.DEFAULT_ENCODING,
+            )
+            self.logger.debug('Extracting README.md from package')
+            local_readme = template_dir / 'README.md'
+            local_readme.write_text(
+                data=data['readme'].replace('\r\n', '\n'),
+                encoding=consts.DEFAULT_ENCODING,
+            )
+            self.logger.debug('Extracting assets from package')
+            for asset_file in assets_dir.rglob('*'):
+                if asset_file.is_file():
+                    target_asset = template_dir / asset_file.relative_to(assets_dir)
+                    target_dir = target_asset.parent
+                    target_dir.mkdir(parents=True, exist_ok=True)
+                    target_asset.write_bytes(asset_file.read_bytes())
+            self.logger.debug('Extracting files from package')
+            for file_item in data.get('files', []):
+                filename = file_item['fileName']
+                content = file_item['content'].replace('\r\n', '\n')
+                target_file = template_dir / filename
+                target_dir = target_file.parent
+                target_dir.mkdir(parents=True, exist_ok=True)
+                target_file.write_text(data=content, encoding=consts.DEFAULT_ENCODING)
+        self.logger.debug('Extracting package done')
+
+    async def watch_project(self, callback, stop_event: asyncio.Event):
+        async for changes in watchfiles.awatch(
+                self.safe_project.template_dir,
+                stop_event=stop_event,
+        ):
+            await callback(
+                change for change in ((change[0], pathlib.Path(change[1])) for change in changes)
+                if self.safe_project.is_template_file(
+                    change[1],
+                    include_descriptor=True,
+                    include_readme=True,
+                )
+            )
+
+    async def update_descriptor(self):
+        try:
+            template_exists = await self.safe_client.check_draft_exists(
+                remote_id=self.remote_id,
+            )
+            if template_exists:
+                self.logger.info('Updating existing remote document template draft %s',
+                                 self.safe_project.safe_template.id)
+                await self.safe_client.update_template_draft(
+                    template=self.safe_project.safe_template,
+                    remote_id=self.remote_id,
+                )
+            else:
+                self.logger.info('Document template draft %s does '
+                                 'not exist on remote - full sync',
+                                 self.safe_project.safe_template.id)
+                await self.store_remote(force=False)
+        except WizardCommunicationError as e:
+            self.logger.error('Failed to update document template draft %s: %s',
+                              self.safe_project.safe_template.id, e.message)
+        except Exception as e:
+            self.logger.error('Failed to update document template draft %s: %s',
+                              self.safe_project.safe_template.id, e)
+
+    async def delete_file(self, filepath: pathlib.Path):
+        if not filepath.is_file():
+            self.logger.debug('%s is not a regular file - skipping',
+                              filepath.as_posix())
+            return
+        try:
+            file = self.safe_project.get_template_file(filepath=filepath)
+            if file is None:
+                self.logger.info('File %s not tracked currently - skipping',
+                                 filepath.as_posix())
+                return
+            await self._delete_template_file(file=file, project_update=True)
+        except Exception as e:
+            self.logger.error('Failed to delete file %s: %s',
+                              filepath.as_posix(), e)
+
+    async def update_file(self, filepath: pathlib.Path):
+        if not filepath.is_file():
+            self.logger.debug('%s is not a regular file - skipping',
+                              filepath.as_posix())
+            return
+        try:
+            remote_file = self.safe_project.get_template_file(filepath=filepath)
+            local_file = self.safe_project.load_file(filepath=filepath)
+            if remote_file is not None:
+                await self._update_template_file(remote_file, local_file, project_update=True)
+            else:
+                await self._create_template_file(file=local_file, project_update=True)
+        except Exception as e:
+            self.logger.error('Failed to update file %s: %s', filepath.as_posix(), e)
+
+    async def process_changes(self, changes: list[ChangeItem], force: bool):
+        self.changes_processor.clear()
+        try:
+            await self.changes_processor.process_changes(changes, force)
+        except Exception as e:
+            self.logger.error('Failed to process changes: %s', e)
+
+
+class ChangesProcessor:
+
+    def __init__(self, tdk: TDKCore):
+        self.tdk: TDKCore = tdk
+        self.descriptor_change: ChangeItem | None = None
+        self.readme_change: ChangeItem | None = None
+        self.file_changes: list[ChangeItem] = []
+
+    def clear(self):
+        self.descriptor_change = None
+        self.readme_change = None
+        self.file_changes = []
+
+    def _split_changes(self, changes: list[ChangeItem]):
+        for change in changes:
+            if change[1] == self.tdk.safe_project.descriptor_path:
+                self.descriptor_change = change
+            elif change[1] == self.tdk.safe_project.used_readme:
+                self.readme_change = change
+            elif self.tdk.safe_project.is_template_file(change[1]):
+                self.file_changes.append(change)
+
+    async def _process_file_changes(self):
+        deleted = set()
+        updated = set()
+        for file_change in self.file_changes:
+            self.tdk.logger.debug('Processing: %s',
+                                  _change(file_change, self.tdk.safe_project.template_dir))
+            change_type = file_change[0]
+            filepath = file_change[1]
+            if change_type == watchfiles.Change.deleted and filepath not in deleted:
+                self.tdk.logger.debug('Scheduling delete operation')
+                deleted.add(filepath)
+                await self.tdk.delete_file(filepath)
+            elif filepath not in updated:
+                self.tdk.logger.debug('Scheduling update operation')
+                updated.add(filepath)
+                await self.tdk.update_file(filepath)
+
+    async def _reload_descriptor(self, force: bool) -> bool:
+        if self.descriptor_change is None:
+            return False
+        if self.descriptor_change[0] == watchfiles.Change.deleted:
+            raise RuntimeError(f'Deleted {self.tdk.safe_project.descriptor_path} ... the end')
+        self.tdk.logger.debug('Reloading %s file', TemplateProject.TEMPLATE_FILE)
+        previous_id = self.tdk.safe_project.safe_template.id
+        self.tdk.safe_project.load_descriptor()
+        self.tdk.safe_project.load_readme()
+        new_id = self.tdk.safe_project.safe_template.id
+        if new_id != previous_id:
+            self.tdk.logger.warning('Template ID changed from %s to %s',
+                                    previous_id, new_id)
+            self.tdk.safe_project.load()
+            await self.tdk.store_remote(force=force)
+            self.tdk.logger.info('Template fully reloaded... waiting for new changes')
+            return True
+        return False
+
+    async def _reload_readme(self) -> bool:
+        if self.readme_change is None:
+            return False
+        if self.readme_change[0] == watchfiles.Change.deleted:
+            raise RuntimeError(f'Deleted used README file {self.tdk.safe_project.used_readme}')
+        self.tdk.logger.debug('Reloading README file')
+        self.tdk.safe_project.load_readme()
+        return True
+
+    async def _update_descriptor(self):
+        if self.readme_change is not None or self.descriptor_change is not None:
+            self.tdk.logger.debug('Updating template descriptor (metadata)')
+            await self.tdk.update_descriptor()
+            self.tdk.safe_project.template = self.tdk.safe_template
+
+    async def process_changes(self, changes: list[ChangeItem], force: bool):
+        self._split_changes(changes)
+        full_reload = await self._reload_descriptor(force)
+        if not full_reload:
+            await self._reload_readme()
+            await self._update_descriptor()
+            await self._process_file_changes()
+        self.tdk.logger.info('All changes processed... waiting for new changes')