pyannotators-entityfishing 0.6.3__tar.gz

pyannotators_entityfishing-0.6.3/.gitignore

@@ -0,0 +1,123 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+results.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Specific
+.idea/
+.groovylintrc.json
+.emailNotif
+uv.lock

pyannotators_entityfishing-0.6.3/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim-bookworm
+# Install prerequisites
+RUN apt-get update -y && \
+    apt-get install -y \
+    gcc && \
+    apt-get install -y --no-install-recommends \
+    g++ \
+    git && \
+# Final upgrade + clean
+    apt-get update -y && \
+    apt-get clean all -y
+
+# Install uv
+RUN pip install uv
+
+# Add pyproject.toml + README.md for uv install
+ADD pyproject.toml pyproject.toml
+ADD README.md README.md

pyannotators_entityfishing-0.6.3/Jenkinsfile

@@ -0,0 +1,386 @@
+pipeline {
+  environment {
+    PATH_HOME = '/home/jenkins'
+    TEST_REPORT_DIR = '/root/test-reports'
+    PYTHONPYCACHEPREFIX = '/tmp/.pytest_cache'
+    PYTHONDONTWRITEBYTECODE = '1'
+    JENKINS_UIDGID = '1004:1004'
+
+    MAJOR_VERSION = '0'
+    MINOR_VERSION = '6'
+  }
+
+  agent none
+
+  triggers {
+    upstream(upstreamProjects: 'pymultirole_plugins/' + BRANCH_NAME.replaceAll('/', '%2F'),\
+                                threshold: hudson.model.Result.SUCCESS)
+  }
+
+  stages {
+    stage('Catch build termination') {
+      agent {
+        node {
+          label 'built-in'
+          customWorkspace "${PATH_HOME}/${JOB_NAME}"
+        }
+      }
+      stages {
+        stage('Analyse build cause') {
+          steps {
+            script {
+              analyseBuildCause()
+            }
+          }
+        }
+      }
+    }
+
+    stage('Generate new version') {
+      when {
+        environment name: 'SKIP_JOB', value: '0'
+      }
+
+      agent {
+        node {
+          label 'built-in'
+          customWorkspace "${PATH_HOME}/${JOB_NAME}"
+        }
+      }
+
+      stages {
+        stage('Add credentials') {
+          steps {
+            script {
+              // Add password file for uv publishing
+              sh "cp ${PATH_HOME}/.passwd-pypi .env"
+            }
+          }
+        }
+
+        stage('Commit new version') {
+          steps {
+            script {
+              println("attempt to publish ${JOB_NAME} with version: ${MAJOR_VERSION}.${MINOR_VERSION}.${BUILD_ID}")
+
+              // push updates of file __init__.py
+              withCredentials([gitUsernamePassword(credentialsId: 'bitbucket-user', gitToolName: 'git-tool')]) {
+                sh 'git pull'
+                sh "echo '\"\"\"Annotator based on entity-fishing\"\"\"' > src/pyannotators_entityfishing/__init__.py"
+                sh "echo '' >> src/pyannotators_entityfishing/__init__.py"
+                sh "echo '__version__ = \"${MAJOR_VERSION}.${MINOR_VERSION}.${BUILD_ID}\"' >> src/pyannotators_entityfishing/__init__.py"
+                sh 'git commit src/pyannotators_entityfishing/__init__.py -m "[Jenkins CI] Commit on version files" || echo "No changes to commit"'
+                sh 'git push'
+              }
+            }
+          }
+        }
+      }
+    }
+
+    stage('Build, test and publish') {
+      when {
+        beforeAgent true
+        environment name: 'SKIP_JOB', value: '0'
+      }
+
+      agent {
+        // dockerfile agent
+        // Mounted volume for Junit reports
+        //   - docker: /root/test-reports
+        //   - host  : /tmp/_${JOB_NAME}/test-reports
+        dockerfile {
+          label 'built-in'
+          customWorkspace "${PATH_HOME}/${JOB_NAME}"
+          filename 'Dockerfile'
+          args "-u root --privileged -v /tmp/_${JOB_NAME}/test-reports:${TEST_REPORT_DIR}"
+        }
+      }
+
+      stages {
+        stage('Install dependencies') {
+          steps {
+            sh 'pip install uv'
+            sh 'uv sync --extra test'
+          }
+        }
+
+        stage('Test & lint python code') {
+          steps {
+            // remove any previous results.xml file
+            sh "rm -f ${TEST_REPORT_DIR}/results.xml"
+            sh 'uv run ruff check .'
+            sh 'uv run ruff format --check .'
+            sh "uv run pytest --junit-xml=${TEST_REPORT_DIR}/results.xml"
+          }
+        }
+
+        stage('Publish on PyPI') {
+          environment {
+            UV_PUBLISH_USERNAME = getUserName '.env'
+            UV_PUBLISH_PASSWORD = getUserPass '.env'
+          }
+          steps {
+            // remove any previous folder dist
+            sh 'rm -rf dist'
+            // pull recent updates of file __init__.py
+            withCredentials([gitUsernamePassword(credentialsId: 'bitbucket-user', gitToolName: 'git-tool')]) {
+              sh 'git config --global pull.rebase false'
+              sh "git config --global --add safe.directory ${WORKSPACE}"
+              sh 'git pull'
+            }
+            // put back owner of .git folder
+            sh "chown -R ${JENKINS_UIDGID} ${WORKSPACE}/.git"
+            // put back owner of pulled file
+            sh "chown ${JENKINS_UIDGID} src/pyannotators_entityfishing/__init__.py"
+            // get git status
+            sh 'git status'
+            // publish on PyPI
+            sh '''
+               export COMMIT_VERSION=$( cat src/pyannotators_entityfishing/__init__.py|grep version|cut -d '"' -f2|tr -s '[:blank:]' )
+               export BUILD_VERSION="${MAJOR_VERSION}"."${MINOR_VERSION}"."${BUILD_ID}"
+               if [ "${COMMIT_VERSION}" = "${BUILD_VERSION}" ] ; then uv build && uv publish ; fi
+               '''
+            // remove current folder dist
+            sh 'rm -rf dist'
+            // remove current folder .hypothesis
+            sh 'rm -rf .hypothesis'
+          }
+        }
+      }
+    }
+  }
+
+  post {
+    // only triggered when blue or green sign
+    success {
+      // node is specified here to get an agent
+      node('built-in') {
+        // keep using customWorkspace to store Junit report
+        ws("${PATH_HOME}/${JOB_NAME}") {
+          script {
+            try {
+              sh 'rm -f results.xml'
+              sh "cp /tmp/_${JOB_NAME}/test-reports/results.xml results.xml"
+            } catch (Exception e) {
+              println 'Exception occurred: ' + e.toString()
+            }
+            try {
+              junit 'results.xml'
+            } catch (Exception e) {
+              println 'Exception occurred: ' + e.toString()
+            }
+            if (sendEmailNotif("${PATH_HOME}/${JOB_NAME}", "${BUILD_NUMBER}")) {
+              println 'sending Success Build notification'
+              def CUSTOM_SUBJECT = '[CI - Jenkinzz SUCCESS] ' + CUSTOM_SUBJECT
+              emailext(
+                  mimeType: 'text/html',
+                  subject: CUSTOM_SUBJECT,
+                  body: '${DEFAULT_CONTENT}',
+                  replyTo: '${DEFAULT_REPLYTO}',
+                  to: '${ADMIN_RECIPIENTS}' + ';' + CUSTOM_RECIPIENTS
+              )
+              switchEmailNotif(false, BUILD_NUMBER)
+            } else {
+              println 'preventing Success Build notification'
+            }
+          }
+        }
+      }
+    }
+    // triggered when red sign
+    failure {
+      // node is specified here to get an agent
+      node('built-in') {
+        // keep using customWorkspace to store Junit report
+        ws("${PATH_HOME}/${JOB_NAME}") {
+          script {
+            try {
+              sh 'rm -f results.xml'
+              sh "cp /tmp/_${JOB_NAME}/test-reports/results.xml results.xml"
+            } catch (Exception e) {
+              println 'Exception occurred: ' + e.toString()
+            }
+            try {
+              junit 'results.xml'
+            } catch (Exception e) {
+              println 'Exception occurred: ' + e.toString()
+            }
+            println 'sending Failure Build notification'
+            def CUSTOM_SUBJECT = '[CI - Jenkinzz FAILURE] ' + CUSTOM_SUBJECT
+            emailext(
+                mimeType: 'text/html',
+                subject: CUSTOM_SUBJECT,
+                body: '${DEFAULT_CONTENT}',
+                replyTo: '${DEFAULT_REPLYTO}',
+                to: '${ADMIN_RECIPIENTS}' + ';' + CUSTOM_RECIPIENTS
+            )
+          }
+        }
+      }
+    }
+    // triggered when black sign
+    aborted {
+      println 'post-declarative message: abort job'
+    }
+  }
+}
+
+// return UV_PUBLISH_USERNAME from given file
+def getUserName(path) {
+  def USERNAME = sh(
+                 script: "grep FLIT_USERNAME ${path}|cut -d '=' -f2",
+                 returnStdout: true
+               ).trim()
+  return USERNAME
+}
+
+// return UV_PUBLISH_PASSWORD from given file
+def getUserPass(path) {
+  def USERPASS = sh(
+                 script: "grep FLIT_PASSWORD ${path}|cut -d '=' -f2",
+                 returnStdout: true
+               ).trim()
+  return USERPASS
+}
+
+// create/remove emailNotif file to trigger email notification
+def switchEmailNotif(toggle, build) {
+  if (toggle) {
+    sh 'echo ' + build + ' > .emailNotif'
+  } else {
+    if (build == BUILD_NUMBER) {
+      sh 'rm -f .emailNotif'
+    }
+  }
+}
+
+// return true if emailNotif file present
+boolean sendEmailNotif(path, build) {
+  def emailNotif = sh(
+                 script: "find ${path} -name '.emailNotif'|wc -l",
+                 returnStdout: true
+               ).trim()
+  def emailContent = ''
+  if (emailNotif == '1') {
+    emailContent = sh(
+                     script: "cat ${path}/.emailNotif",
+                     returnStdout: true
+                   ).trim()
+  }
+  return (emailContent == build)
+}
+
+def analyseBuildCause() {
+  String[] upstreamProjects = ['pyimporters_plugins']
+  boolean upstreamRunning = false
+  String jobName
+  // iterate over upstreamProjects
+  for (upstream_project in upstreamProjects) {
+    Jenkins.instance.getItemByFullName(upstream_project).items.each { repository ->
+      boolean isRunning = false
+      if ( repository.name == BRANCH_NAME ) {
+        repository.allJobs.each { job ->
+          job.builds.each { build ->
+            if ( build.result == (null) ) {
+              jobName = build.parent.parent.name
+              isRunning = true
+            }
+          }
+          if ( isRunning ) {
+            upstreamRunning = true
+          }
+        }
+      }
+    }
+  }
+
+  boolean lastCommitIsTeam = sh(
+    script: 'git log -1 | grep "\\[Jenkins CI\\]"',
+    returnStatus: true
+  )
+
+  if (upstreamRunning) {
+    println 'Skipping build because upstream job detected (' + jobName + ')'
+    env.SKIP_JOB = '1'
+    switchEmailNotif(false, 0)
+    currentBuild.result = 'NOT_BUILT'
+  }
+
+  boolean isStartedByUser = currentBuild.rawBuild.getCause(hudson.model.Cause$UserIdCause) != null
+  if (isStartedByUser && !upstreamRunning) {
+    env.SKIP_JOB = '0'
+    env.CUSTOM_SUBJECT = JOB_NAME + ' - Manual Build #' + BUILD_NUMBER
+    env.CUSTOM_RECIPIENTS = emailextrecipients([[$class: 'RequesterRecipientProvider']])
+    switchEmailNotif(true, BUILD_NUMBER)
+    println 'Job started by User, proceeding'
+  }
+
+  boolean isStartedByUpstream = currentBuild.rawBuild.getCause(hudson.model.Cause$UpstreamCause) != null
+  if (isStartedByUpstream && !upstreamRunning) {
+    int changeSetCount = 0
+    int ciSkipCount = 0
+    String upstreamFullJobName = ''
+    for (Run upstreamBuild : currentBuild.upstreamBuilds) {
+      upstreamFullJobName   = upstreamBuild.rawBuild.fullDisplayName
+      if (upstreamBuild.changeSets != null) {
+        def changeLogSets = upstreamBuild.changeSets
+        for (int i = 0; i < changeLogSets.size(); i++) {
+          changeSetCount++
+          def entries = changeLogSets[i].items
+          for (int j = 0; j < entries.length; j++) {
+            def entry = entries[j]
+            if (entry.msg.contains('[Jenkins CI]')) {
+              ciSkipCount++
+            }
+          }
+        }
+      }
+    }
+    if (changeSetCount > 0 && changeSetCount == ciSkipCount) {
+      env.SKIP_JOB = '1'
+      switchEmailNotif(false, 0)
+      println 'Job started by Upstream [' + upstreamFullJobName + '], with CI commit, skipping'
+      currentBuild.result = 'NOT_BUILT'
+    } else {
+      env.SKIP_JOB = '0'
+      env.CUSTOM_SUBJECT = JOB_NAME + ' - Upstream Build #' + BUILD_NUMBER
+      env.CUSTOM_RECIPIENTS = emailextrecipients([[$class:'UpstreamComitterRecipientProvider']])
+      switchEmailNotif(true, BUILD_NUMBER)
+      println 'Job started by Upstream [' + upstreamFullJobName + '], proceeding'
+    }
+  }
+
+  boolean isStartedByCommit = currentBuild.rawBuild.getCause(jenkins.branch.BranchEventCause) != null
+  if (isStartedByCommit && lastCommitIsTeam && !upstreamRunning) {
+    env.SKIP_JOB = '0'
+    env.CUSTOM_SUBJECT = JOB_NAME + ' - SCM Build #' + BUILD_NUMBER
+    env.CUSTOM_RECIPIENTS = emailextrecipients([[$class: 'DevelopersRecipientProvider'], [$class:'CulpritsRecipientProvider']])
+    switchEmailNotif(true, BUILD_NUMBER)
+    println 'Job started by User Commit, proceeding'
+  }
+
+  boolean isStartedByCron = currentBuild.rawBuild.getCause(hudson.triggers.TimerTrigger$TimerTriggerCause) != null
+  if (isStartedByCron && lastCommitIsTeam && !upstreamRunning) {
+    env.SKIP_JOB = '0'
+    env.CUSTOM_SUBJECT = JOB_NAME + ' - CRON Build #' + BUILD_NUMBER
+    env.CUSTOM_RECIPIENTS = emailextrecipients([[$class: 'DevelopersRecipientProvider'], [$class:'CulpritsRecipientProvider']])
+    switchEmailNotif(true, BUILD_NUMBER)
+    println 'Job started by Cron, proceeding'
+  }
+
+  boolean isStartedByBranchDiscovery = currentBuild.rawBuild.getCause(jenkins.branch.BranchIndexingCause) != null
+  if (isStartedByBranchDiscovery && lastCommitIsTeam && !upstreamRunning) {
+    env.SKIP_JOB = '0'
+    env.CUSTOM_SUBJECT = JOB_NAME + ' - BranchDiscovery Build #' + BUILD_NUMBER
+    env.CUSTOM_RECIPIENTS = emailextrecipients([[$class: 'DevelopersRecipientProvider'], [$class:'CulpritsRecipientProvider']])
+    switchEmailNotif(true, BUILD_NUMBER)
+    println 'Job started by Branch Discovery, proceeding'
+  }
+
+  if (!lastCommitIsTeam && !upstreamRunning && !isStartedByUser && !isStartedByUpstream) {
+    println 'Skipping build because last commit has been done by CI'
+    env.SKIP_JOB = '1'
+    switchEmailNotif(false, 0)
+  }
+}

pyannotators_entityfishing-0.6.3/MIGRATION.md

@@ -0,0 +1,82 @@
+# Migration from rest-pysuggester EntityFishing to pyannotators_entityfishing
+
+This document describes the migration from the `rest-pysuggester` `EntityFishingSuggester`
+to the standalone `pyannotators_entityfishing` annotator plugin.
+
+## Architecture change
+
+The original implementation lived inside `rest-pysuggester` as a `Suggester` subclass
+with tight coupling to MongoDB, the Sherpa training pipeline, and a shared resource
+manager. The new implementation is a standalone `AnnotatorBase` plugin following the
+`pymultirole_plugins` v1 contract.
+
+### What was kept (upward-compatible)
+
+| Feature | Original | New |
+|---------|----------|-----|
+| EntityFishing HTTP client | `ef_client.py` (env-var config) | `ef_client.py` (constructor params from `EntityFishingParameters`) |
+| Disambiguation parameters | `minSelectorScore`, `maxTermFrequency` | Same field names and defaults |
+| Label mapping | `mapped_labels` (dict of JSON mongoquery) | Same format, same `mongoquery` library |
+| Default label | `default_label` | Same |
+| Output label filtering | `output_labels` | Same |
+| Fingerprinting | `fingerprint`, `wikidata_properties`, `multivalued_props` | Same |
+| Chunking | `do_chunking` | Same |
+| Short text mode | `short_text` | Same |
+| Noun filtering | `noun_forms_only` (default True) | Same (default changed to False, spacy optional) |
+| Overlap filtering | `filter_annotations` (longest match) | Same algorithm |
+| Annotation output | `Annotation` with `Term(identifier=QID, lexicon="wikidata", ...)` | Same schema |
+| Parallel requests | `requests-futures` / `FuturesSession` | Same |
+| Caching | `requests-cache` / `CachedSession` | Same (disabled by default) |
+
+### What changed
+
+1. **No training pipeline**: The annotator is stateless. There is no `train_model_unsafe`
+   or model persistence. The label classes are derived from `mapped_labels` +
+   `default_label` + `output_labels` at annotation time.
+
+2. **No MongoDB integration**: Documents are passed in and returned directly.
+
+3. **No spacy by default**: `noun_forms_only` defaults to `False` (was `True`).
+   When enabled, spacy is imported lazily and a language-specific model is loaded.
+   Install spacy separately: `pip install pyannotators-entityfishing[spacy]`.
+
+4. **Configuration via parameters**: The EntityFishing service URL, pool size, and
+   caching settings are now fields on `EntityFishingParameters` instead of environment
+   variables. This makes the annotator self-contained and testable.
+
+5. **Classification mode not ported**: The `EntityFishingClassifierSuggester` and
+   `EntityFishingDispatcherSuggester` were not ported. The classifier mode was
+   incomplete in the original and can be added later if needed.
+
+6. **Python 3.12+ / Pydantic v2**: Following the migration guide from
+   `pyannotators_duckling`, the project uses modern Python type syntax and Pydantic v2.
+
+## Parameter mapping reference
+
+| Original `EntityFishingSequenceTrainParameters` | New `EntityFishingParameters` |
+|--------------------------------------------------|-------------------------------|
+| `minSelectorScore: float = 0.3` | `minSelectorScore: float = 0.3` |
+| `maxTermFrequency: Optional[float]` | `maxTermFrequency: float \| None` |
+| `mapped_labels: Dict[str, str]` | `mapped_labels: dict[str, str] \| None` |
+| `default_label: Optional[str]` | `default_label: str \| None` |
+| `output_labels: List[str]` | `output_labels: list[str] \| None` |
+| `noun_forms_only: Optional[bool] = True` | `noun_forms_only: bool = False` |
+| `fingerprint: Optional[str]` | `fingerprint: str \| None` |
+| `wikidata_properties: Optional[str]` | `wikidata_properties: str \| None` |
+| `do_chunking: Optional[bool] = False` | `do_chunking: bool = False` |
+| `short_text: Optional[bool] = False` | `short_text: bool = False` |
+| `multivalued_props: Optional[bool] = False` | `multivalued_props: bool = False` |
+| *(env)* `APP_EF_URI` | `ef_uri: str` |
+| *(env)* `APP_EF_CLIENT_POOLSIZE` | `ef_pool_size: int` |
+| *(env)* `APP_EF_CLIENT_FULL` | `ef_full: bool` |
+
+## Build and tooling
+
+Following `pyannotators_duckling` MIGRATION.md:
+
+- **Build system**: hatchling + uv
+- **Python**: >=3.12
+- **Pydantic**: v2
+- **Linter**: ruff
+- **Tests**: `uv run pytest`
+- **CI**: Jenkinsfile with `uv sync --extra test`