media-tagging 0.2.0.dev1__tar.gz

media-tagging-0.2.0.dev1/PKG-INFO

@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: media-tagging
+Version: 0.2.0.dev1
+Author: Google Inc. (gTech gPS CSE team)
+Author-email: no-reply@google.com
+License: Apache 2.0
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Operating System :: OS Independent
+Classifier: License :: OSI Approved :: Apache Software License
+Description-Content-Type: text/markdown

media-tagging-0.2.0.dev1/README.md

@@ -0,0 +1,48 @@
+# Welltech Media Tagging
+
+## Prerequisites
+
+* Google Cloud project with billing enabled.
+* [Video Intelligence API](https://console.cloud.google.com/apis/library/videointelligence.googleapis.com) and [Vision API](https://console.cloud.google.com/apis/library/vision.googleapis.com) enabled.
+* Python3.8+
+* Access to repository configured. In order to clone this repository you need
+	to do the following:
+	*   Visit https://professional-services.googlesource.com/new-password and
+			login with your account.
+    * Once authenticated please copy all lines in box
+        and paste them in the terminal.
+
+
+## Run
+
+
+1. Install `media-tagger`
+
+```
+pip install media-tagging
+```
+
+2. Perform tagging
+
+```
+media-tagger --media-path MEDIA_PATH --tagger TAGGER_TYPE --writer WRITER_TYPE
+```
+where:
+* MEDIA_PATH - comma-separated names of files for tagging (can be urls).
+* TAGGER_TYPE - name of tagger, supported options:
+  * `vision-api` - tags images based on [Google Cloud Vision API](https://cloud.google.com/vision/),
+  * `video-api` for videos based on [Google Cloud Video Intelligence API](https://cloud.google.com/video-intelligence/)
+  * `gemini-image` - Uses Gemini to tags images. Add `--tagger.n_tags=<N_TAGS>`
+     parameter to control number of tags returned by tagger.
+  * `gemini-structured-image`  - Uses Gemini to find certain tags in the images.
+    Add `--tagger.tags='tag1, tag2, ..., tagN` parameter to find certain tags
+    in the image.
+  * `gemini-description-image` - Provides brief description of the image,
+* WRITER_TYPE - name of writer, one of `csv`, `json`
+
+By default script will create a single file with tagging results for each media_path.
+If you want to combine results into a single file add `--output OUTPUT_NAME` flag (without extension, i.e. `--output tagging_sample`.
+
+
+## Disclaimer
+This is not an officially supported Google product.

media-tagging-0.2.0.dev1/entrypoints/cli.py

@@ -0,0 +1,93 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Provides CLI for media tagging."""
+
+import argparse
+import logging
+import os
+
+import smart_open
+from gaarf.cli import utils as gaarf_utils
+
+from media_tagging import tagger, utils, writer
+from media_tagging.taggers import base as base_tagger
+
+
+def tag_media(
+  media_path: str | os.PathLike,
+  tagger_type: base_tagger.BaseTagger,
+  writer_type: writer.BaseWriter = writer.JsonWriter(),
+  single_output_name: str | None = None,
+  tagging_parameters: dict[str, str] | None = None,
+) -> None:
+  """Runs media tagging algorithm.
+
+  Args:
+    media_path: Local or remote path to media file.
+    tagger_type: Initialized tagger.
+    writer_type: Initialized writer for saving tagging results.
+    single_output_name: Parameter for saving results to a single file.
+    tagging_parameters: Optional keywords arguments to be sent for tagging.
+  """
+  media_paths = media_path.split(',')
+  if not tagging_parameters:
+    tagging_parameters = {}
+  results = []
+  for path in media_paths:
+    media_name = utils.convert_path_to_media_name(path)
+    logging.info('Processing media: %s', path)
+    with smart_open.open(path, 'rb') as f:
+      media_bytes = f.read()
+    results.append(
+      tagger_type.tag(
+        media_name,
+        media_bytes,
+        tagging_options=base_tagger.TaggingOptions(**tagging_parameters),
+      )
+    )
+  writer_type.write(results, single_output_name)
+
+
+def main():
+  """Main entrypoint."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--media-path', dest='media_path')
+  parser.add_argument('--tagger', dest='tagger', default='vision-api')
+  parser.add_argument('--writer', dest='writer', default='json')
+  parser.add_argument('--output-to-file', dest='output', default=None)
+  parser.add_argument('--loglevel', dest='loglevel', default='INFO')
+  args, kwargs = parser.parse_known_args()
+
+  concrete_tagger = tagger.create_tagger(args.tagger)
+  concrete_writer = writer.create_writer(args.writer)
+  tagging_parameters = gaarf_utils.ParamsParser(['tagger']).parse(kwargs)
+
+  logging.basicConfig(
+    format='[%(asctime)s][%(name)s][%(levelname)s] %(message)s',
+    level=args.loglevel,
+    datefmt='%Y-%m-%d %H:%M:%S',
+  )
+  logging.getLogger(__file__)
+
+  tag_media(
+    media_path=args.media_path,
+    tagger_type=concrete_tagger,
+    writer_type=concrete_writer,
+    single_output_name=args.output,
+    tagging_parameters=tagging_parameters.get('tagger'),
+  )
+
+
+if __name__ == '__main__':
+  main()

media-tagging-0.2.0.dev1/entrypoints/server.py

@@ -0,0 +1,101 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Provides HTTP endpoint for media tagging."""
+
+import logging
+
+import fastapi
+import smart_open
+from typing_extensions import TypedDict
+
+from media_tagging import tagger, utils
+from media_tagging.tagger import base as base_tagger
+
+taggers: dict[str, base_tagger.BaseTagger] = {}
+app = fastapi.FastAPI()
+
+
+class MediaPostRequest(TypedDict):
+  """Specifies structure of request for tagging media.
+
+  Attributes:
+    media_url: Local or remote URL of media.
+  """
+
+  media_url: str
+  tagging_parameters: dict[str, int | list[str]]
+
+
+@app.post('/tagger/llm')
+async def tag_with_llm(
+  data: MediaPostRequest = fastapi.Body(embed=True),
+) -> dict[str, str]:
+  """Performs media tagging via LLMs.
+
+  Args:
+    data: Post request for media tagging.
+
+  Returns:
+    Json results of tagging.
+  """
+  if not (llm_tagger := taggers.get('gemini-image')):
+    llm_tagger = tagger.create_tagger('gemini-image')
+    taggers['gemini-image'] = llm_tagger
+  if media_url := data.get('media_url'):
+    media_name = utils.convert_path_to_media_name(media_url)
+    logging.info('Processing media: %s', media_url)
+    with smart_open.open(media_url, 'rb') as f:
+      media_bytes = f.read()
+    tagging_options = base_tagger.TaggingOptions(
+      **data.get('tagging_parameters')
+    )
+    tagging_result = llm_tagger.tag(
+      name=media_name, content=media_bytes, tagging_options=tagging_options
+    )
+    return fastapi.responses.JSONResponse(
+      content=fastapi.encoders.jsonable_encoder(tagging_result.dict())
+    )
+  raise ValueError('No path to media is provided.')
+
+
+@app.post('/tagger/api')
+async def tag_with_api(
+  data: MediaPostRequest = fastapi.Body(embed=True),
+) -> dict[str, str]:
+  """Performs media tagging via Google Cloud APIs.
+
+  Args:
+    data: Post request for media tagging.
+
+  Returns:
+    Json results of tagging.
+  """
+  if not (api_tagger := taggers.get('vision-api')):
+    api_tagger = tagger.create_tagger('vision-api')
+    taggers['vision-api'] = api_tagger
+  if media_url := data.get('media_url'):
+    media_name = utils.convert_path_to_media_name(media_url)
+    logging.info('Processing media: %s', media_url)
+    with smart_open.open(media_url, 'rb') as f:
+      media_bytes = f.read()
+    tagging_options = base_tagger.TaggingOptions(
+      **data.get('tagging_parameters')
+    )
+    tagging_result = api_tagger.tag(
+      name=media_name, content=media_bytes, tagging_options=tagging_options
+    )
+    return fastapi.responses.JSONResponse(
+      content=fastapi.encoders.jsonable_encoder(tagging_result.dict())
+    )
+  raise ValueError('No path to media is provided.')

media-tagging-0.2.0.dev1/media_tagging/llms.py

@@ -0,0 +1,63 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for defining various LLMs."""
+
+from __future__ import annotations
+
+import os
+
+import langchain_google_genai as genai
+from langchain_core import language_models
+
+_GEMINI_SAFETY_SETTINGS: dict[genai.HarmCategory, genai.HarmBlockThreshold] = {
+  genai.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: genai.HarmBlockThreshold.BLOCK_NONE,  # noqa: E501
+  genai.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: genai.HarmBlockThreshold.BLOCK_NONE,  # noqa: E501
+  genai.HarmCategory.HARM_CATEGORY_HARASSMENT: genai.HarmBlockThreshold.BLOCK_NONE,  # noqa: E501
+}
+
+
+def create_llm(
+  llm_type: str, llm_parameters: dict[str, str] | None = None
+) -> language_models.BaseLanguageModel:
+  """Creates LLM based on type and parameters.
+
+  Args:
+    llm_type: Type of LLM to instantiate.
+    llm_parameters: Various parameters to instantiate LLM.
+
+  Returns:
+    Initialized LLM.
+
+  Raises:
+    InvalidLLMTypeError: When incorrect LLM type is specified.
+  """
+  mapping = {
+    'gemini': genai.ChatGoogleGenerativeAI,
+  }
+  if llm := mapping.get(llm_type):
+    if not llm_parameters:
+      llm_parameters = {}
+    if llm_type == 'gemini':
+      llm_parameters.update(
+        {
+          'safety_settings': _GEMINI_SAFETY_SETTINGS,
+          'google_api_key': os.environ.get('GOOGLE_API_KEY'),
+        }
+      )
+    return llm(**llm_parameters)
+  raise InvalidLLMTypeError(f'Unsupported LLM type: {llm_type}')
+
+
+class InvalidLLMTypeError(Exception):
+  """Error when incorrect LLM type is specified."""

media-tagging-0.2.0.dev1/media_tagging/tagger.py

@@ -0,0 +1,60 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for performing media tagging.
+
+Media tagging sends API requests to tagging engine (i.e. Google Vision API)
+and returns tagging results that can be easily written.
+"""
+
+from media_tagging.taggers import api, base, llm
+
+_TAGGERS = {
+  'vision-api': api.GoogleVisionAPITagger,
+  'video-api': api.GoogleVideoIntelligenceAPITagger,
+  'gemini-image': llm.GeminiImageTagger,
+  'gemini-structured-image': llm.GeminiImageTagger,
+  'gemini-description-image': llm.GeminiImageTagger,
+}
+
+_LLM_TAGGERS_TYPES = {
+  'gemini-image': llm.LLMTaggerTypeEnum.UNSTRUCTURED,
+  'gemini-structured-image': llm.LLMTaggerTypeEnum.STRUCTURED,
+  'gemini-description-image': llm.LLMTaggerTypeEnum.DESCRIPTION,
+}
+
+
+def create_tagger(
+  tagger_type: str, tagger_parameters: dict[str, str] | None = None
+) -> base.BaseTagger:
+  """Factory for creating taggers based on provided type.
+
+  Args:
+    tagger_type: Type of tagger.
+    tagger_parameters: Various parameters to instantiate tagger.
+
+  Returns:
+    Concrete tagger class.
+  """
+  if not tagger_parameters:
+    tagger_parameters = {}
+  if tagger := _TAGGERS.get(tagger_type):
+    if issubclass(tagger, llm.LLMTagger):
+      return tagger(
+        tagger_type=_LLM_TAGGERS_TYPES.get(tagger_type), **tagger_parameters
+      )
+    return tagger(**tagger_parameters)
+  raise ValueError(
+    f'Incorrect tagger {type} is provided, '
+    f'valid options: {list(_TAGGERS.keys())}'
+  )

media-tagging-0.2.0.dev1/media_tagging/taggers/api.py

@@ -0,0 +1,119 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for performing media tagging via Google APIs.
+
+Media tagging sends API requests to tagging engine (i.e. Google Vision API)
+and returns tagging results that can be easily written.
+"""
+
+from collections import defaultdict
+
+from google.cloud import videointelligence, vision
+from typing_extensions import override
+
+from media_tagging.taggers import base
+
+
+class GoogleVisionAPITagger(base.BaseTagger):
+  """Tagger responsible for getting image tags from Cloud Vision API.
+
+  Attributes:
+    client: Vision API client responsible to tagging.
+  """
+
+  def __init__(self, project: str | None = None) -> None:
+    """Initializes GoogleVisionAPITagger with a given project.
+
+    Args:
+      project: Google Cloud project id.
+    """
+    self._project = project
+
+  @property
+  def client(self) -> vision.ImageAnnotatorClient:
+    """Creates ImageAnnotatorClient."""
+    return vision.ImageAnnotatorClient()
+
+  @override
+  def tag(
+    self,
+    name: str,
+    content: bytes,
+    tagging_options: base.TaggingOptions = base.TaggingOptions(),
+    **kwargs: str,
+  ) -> base.TaggingResult:
+    image = vision.Image(content=content)
+    response = self.client.label_detection(image=image)
+    tags = [
+      base.Tag(name=r.description, score=r.score)
+      for r in response.label_annotations
+    ]
+    if n_tags := tagging_options.n_tags:
+      tags = self._limit_number_of_tags(tags, n_tags)
+    return base.TaggingResult(identifier=name, type='image', content=tags)
+
+
+class GoogleVideoIntelligenceAPITagger(base.BaseTagger):
+  """Tagger responsible for getting image tags from Video Intelligence API.
+
+  Attributes:
+    client: Video Intelligence API client responsible to tagging.
+  """
+
+  def __init__(self, project: str | None = None) -> None:
+    """Initializes GoogleVideoIntelligenceAPITagger with a given project.
+
+    Args:
+      project: Google Cloud project id.
+    """
+    self._project = project
+
+  @property
+  def client(self) -> videointelligence.VideoIntelligenceServiceClient:
+    """Creates VideoIntelligenceServiceClient."""
+    return videointelligence.VideoIntelligenceServiceClient()
+
+  @override
+  def tag(
+    self,
+    name: str,
+    content: bytes,
+    tagging_options: base.TaggingOptions = base.TaggingOptions(),
+  ) -> base.TaggingResult:
+    request = videointelligence.AnnotateVideoRequest(
+      input_content=content,
+      video_context=videointelligence.VideoContext(
+        label_detection_config=videointelligence.LabelDetectionConfig(
+          frame_confidence_threshold=0.11,
+          label_detection_mode=(
+            videointelligence.LabelDetectionMode.SHOT_AND_FRAME_MODE
+          ),
+        )
+      ),
+      features=[videointelligence.Feature.LABEL_DETECTION],
+    )
+    operation = self.client.annotate_video(request)
+    response = operation.result(timeout=180)
+    tags_scores: dict[str, float] = defaultdict(float)
+    for frame_label in response.annotation_results[0].frame_label_annotations:
+      tags_scores[frame_label.entity.description] += sum(
+        c.confidence for c in frame_label.frames
+      )
+
+    tags = [
+      base.Tag(name=name, score=score) for name, score in tags_scores.items()
+    ]
+    if n_tags := tagging_options.n_tags:
+      tags = self._limit_number_of_tags(tags, n_tags)
+    return base.TaggingResult(identifier=name, type='video', content=tags)

media-tagging-0.2.0.dev1/media_tagging/taggers/base.py

@@ -0,0 +1,115 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for defining common interface for taggers."""
+
+import abc
+import dataclasses
+from collections.abc import MutableSequence, Sequence
+from typing import Literal
+
+import pydantic
+
+
+@dataclasses.dataclass
+class TaggingOptions:
+  """Specifies options to refine media tagging.
+
+  Attributes:
+    n_tags: Max number of tags to return.
+    tags: Particular tags to find in the media.
+  """
+
+  n_tags: int | None = None
+  tags: Sequence[str] | None = None
+
+  def __post_init__(self):  # noqa: D105
+    if self.tags and not isinstance(self.tags, MutableSequence):
+      self.tags = [tag.strip() for tag in self.tags.split(',')]
+
+
+class Tag(pydantic.BaseModel):
+  """Represents a single tag.
+
+  Attributes:
+    name: Descriptive name of the tag.
+    score: Score assigned to the tag.
+  """
+
+  name: str = pydantic.Field(description='tag_name')
+  score: float = pydantic.Field(description='tag_score from 0 to 1')
+
+
+class Description(pydantic.BaseModel):
+  """Represents brief description of the media.
+
+  Attributes:
+    text: Textual description of the media.
+  """
+
+  text: str = pydantic.Field(description='brief description of the media')
+
+
+class TaggingResult(pydantic.BaseModel):
+  """Contains tagging information for a given identifier.
+
+  Attributes:
+    identifier: Unique identifier of a media being tagged.
+    type: Type of media.
+    tags: Tags associated with a given media.
+  """
+
+  identifier: str = pydantic.Field(description='media identifier')
+  type: Literal['image', 'video'] = pydantic.Field(description='type of media')
+  content: list[Tag] | Description = pydantic.Field(
+    description='tags or description in the result'
+  )
+
+
+class BaseTagger(abc.ABC):
+  """Interface to inherit all taggers from."""
+
+  @abc.abstractmethod
+  def tag(
+    self,
+    name: str,
+    content: bytes,
+    tagging_options: TaggingOptions = TaggingOptions(),
+    **kwargs: str,
+  ) -> TaggingResult:
+    """Sends media bytes to tagging engine.
+
+    Args:
+      name: Identifier of the media content being tagged.
+      content: Raw types of the media.
+      tagging_options: Parameters to refine the tagging results.
+      kwargs: Optional keywords arguments to be sent for tagging.
+
+    Returns:
+      Results of tagging.
+    """
+
+  def _limit_number_of_tags(
+    self, tags: Sequence[Tag], n_tags: int
+  ) -> list[Tag]:
+    """Returns limited number of tags from the pool.
+
+    Args:
+      tags: All tags produced by tagging algorithm.
+      n_tags: Max number of tags to return.
+
+    Returns:
+      Limited number of tags sorted by the score.
+    """
+    sorted_tags = sorted(tags, key=lambda x: x.score, reverse=True)
+    return sorted_tags[:n_tags]