lecrapaud 0.1.0__py3-none-any.whl

lecrapaud/integrations/openai_integration.py

@@ -0,0 +1,225 @@
+import re
+from openai import OpenAI
+import tiktoken
+from lecrapaud.utils import logger
+from lecrapaud.config import OPENAI_API_KEY
+
+# OpenAI’s max tokens per request for embeddings
+MAX_TOKENS = 8192
+OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"
+OPEN_AI_MODEL = "gpt-4o-2024-08-06"
+OPEN_AI_TOKENIZER = "cl100k_base"
+OPEN_AI_EMBEDDING_DIM = 1536  # 3072 if embedding model is text-embedding-3-large
+TPM_LIMIT = 5000000
+TPR_LIMIT = 300_000  # known empirically because of a error message
+MAX_LENGHT_ARRAY_FOR_BULK_EMBEDDINGS = 2048
+
+
+def get_openai_client():
+    if not OPENAI_API_KEY:
+        raise ValueError(
+            "Please set an OPENAI_API_KEY environment variable to use embeddings"
+        )
+    return OpenAI(api_key=OPENAI_API_KEY)
+
+
+def get_openai_embedding(document: str | dict) -> list[float]:
+    """embed a string into a vector using latest openai model, text-embedding-3-small
+
+    :param document: the string to be embedded
+    :return: the embedded vector
+    """
+    client = get_openai_client()
+
+    if isinstance(document, dict):
+        document = dict_to_markdown_headers_nested(document)
+    if not isinstance(document, str):
+        raise ValueError("document must be a string or dict")
+
+    try:
+        res = client.embeddings.create(input=document, model=OPENAI_EMBEDDING_MODEL)
+    except Exception as e:
+        if f"This model's maximum context length is {MAX_TOKENS} tokens" in str(e):
+            raise Exception(
+                f"get_embedding: the document is too long to be vectorized, it is longer than {MAX_TOKENS} tokens"
+            )
+        else:
+            raise Exception(e)
+
+    return res.data[0].embedding
+
+
+def get_openai_embeddings(
+    documents: list[str | dict], dimensions=None
+) -> list[list[float]]:
+    """embed a string into a vector using latest openai model, text-embedding-3-small
+
+    :param document: an array of documents
+    :return: a array of embedded vector
+    """
+    _documents = documents.copy()
+    client = get_openai_client()
+    dimensions = dimensions or OPEN_AI_EMBEDDING_DIM
+
+    if not isinstance(documents, list):
+        raise ValueError("documents must be a list")
+
+    for i, doc in enumerate(documents):
+        if isinstance(doc, dict):
+            doc = dict_to_markdown_headers_nested(doc)
+            _documents[i] = doc
+        if not isinstance(doc, str):
+            raise ValueError("documents must be a list of strings or dict")
+
+    try:
+        max_token = min(max_number_of_tokens(_documents), MAX_TOKENS)
+        docs_per_batch = min(
+            TPM_LIMIT // max_token,
+            TPR_LIMIT // max_token,
+            MAX_LENGHT_ARRAY_FOR_BULK_EMBEDDINGS,
+        )  # TODO: un peu plus de marge ?
+
+        embeddings = []
+        for i, chunk in enumerate(
+            [
+                _documents[i : i + docs_per_batch]
+                for i in range(0, len(_documents), docs_per_batch)
+            ]
+        ):
+            logger.debug(f"Embedding chunk {i+1} with {len(chunk)} documents...")
+            res = client.embeddings.create(
+                input=[doc for doc in chunk],
+                model=OPENAI_EMBEDDING_MODEL,
+                dimensions=dimensions,
+            )
+            chunk_embeddings = [data.embedding for data in res.data]
+            embeddings.extend(chunk_embeddings)
+
+        return embeddings
+
+    except Exception as e:
+        if f"This model's maximum context length is {MAX_TOKENS} tokens" in str(e):
+            raise Exception(
+                f"get_embedding: the document is too long to be vectorized, it is longer than {MAX_TOKENS} tokens"
+            )
+        else:
+            raise Exception(e)
+
+
+def max_number_of_tokens(list):
+    return max([num_tokens_from_string(str(item)) for item in list])
+
+
+def num_tokens_from_string(string: str, encoding_name: str = OPEN_AI_TOKENIZER) -> int:
+    """Count the number of token in string
+
+    :param string: the string
+    :param encoding_name: the encoding model
+    :return: the number of tokens
+    """
+    if not string:
+        return 0
+    tokenizer = tiktoken.get_encoding(encoding_name)
+    num_tokens = len(tokenizer.encode(string))
+    return num_tokens
+
+
+def chunk_text_words(text, max_tokens=MAX_TOKENS):
+    """Splits text into chunks of max_tokens or less."""
+    words = text.split()
+
+    chunks = []
+    current_chunk = []
+    current_tokens = 0
+
+    for word in words:
+        word_tokens = num_tokens_from_string(word)  # Count tokens for word
+        if current_tokens + word_tokens > max_tokens:
+            chunks.append(" ".join(current_chunk))
+            current_chunk = []
+            current_tokens = 0
+
+        current_chunk.append(word)
+        current_tokens += word_tokens
+
+    if current_chunk:
+        chunks.append(" ".join(current_chunk))
+
+    return chunks
+
+
+def chunk_text_sentences(text, max_tokens=MAX_TOKENS):
+    # Sentence-split using regex (can also use nltk.sent_tokenize if preferred)
+    # TODO: should we do a sliding window for chunking ?
+    sentences = re.split(r"(?<=[.!?])\s+", text)
+
+    chunks = []
+    current_chunk = ""
+    current_tokens = 0
+
+    for sentence in sentences:
+        sentence_tokens = num_tokens_from_string(sentence)
+
+        if current_tokens + sentence_tokens <= max_tokens:
+            current_chunk += " " + sentence if current_chunk else sentence
+            current_tokens += sentence_tokens
+        else:
+            if current_chunk:
+                chunks.append(current_chunk.strip())
+            # Sentence too long to fit, need to split it
+            if sentence_tokens > max_tokens:
+                words = sentence.split()
+                sub_chunk = ""
+                sub_tokens = 0
+                for word in words:
+                    word_tokens = num_tokens_from_string(word + " ")
+                    if sub_tokens + word_tokens > max_tokens:
+                        chunks.append(sub_chunk.strip())
+                        sub_chunk = word
+                        sub_tokens = word_tokens
+                    else:
+                        sub_chunk += " " + word if sub_chunk else word
+                        sub_tokens += word_tokens
+                if sub_chunk:
+                    chunks.append(sub_chunk.strip())
+                current_chunk = ""
+                current_tokens = 0
+            else:
+                current_chunk = sentence
+                current_tokens = sentence_tokens
+
+    if current_chunk:
+        chunks.append(current_chunk.strip())
+
+    return chunks
+
+
+def truncate_text(text, max_tokens=MAX_TOKENS):
+    """Limits text to max_tokens or less by truncating."""
+    words = text.split()
+    truncated_text = []
+    current_length = 0
+
+    for word in words:
+        token_length = num_tokens_from_string(word)  # Count tokens for word
+        if current_length + token_length > max_tokens:
+            break  # Stop once limit is reached
+
+        truncated_text.append(word)
+        current_length += token_length
+
+    return " ".join(truncated_text)
+
+
+def dict_to_markdown_headers_nested(d: dict, level: int = 1) -> str:
+    lines = []
+    for key, value in d.items():
+        header = "#" * level + f" {key}"
+        if isinstance(value, dict):
+            lines.append(header)
+            lines.append(dict_to_markdown_headers_nested(value, level + 1))
+        else:
+            lines.append(header)
+            lines.append(str(value).strip())
+        lines.append("")  # Blank line between sections
+    return "\n".join(lines)

lecrapaud/jobs/__init__.py

@@ -0,0 +1,13 @@
+from celery import Celery, signals
+from lecrapaud.jobs import config
+from lecrapaud.utils import setup_logger
+
+
+@signals.setup_logging.connect
+def configure_celery_logging(**kwargs):
+    setup_logger()
+
+
+app = Celery("src")
+app.config_from_object(config)
+app.autodiscover_tasks(["src.jobs"])

lecrapaud/jobs/config.py

@@ -0,0 +1,17 @@
+from lecrapaud.config import REDIS_URL
+
+REDIS_URL = REDIS_URL + "/1"
+broker_url = REDIS_URL
+result_backend = REDIS_URL
+
+# For RedBeat
+redbeat_redis_url = REDIS_URL
+beat_scheduler = "redbeat.RedBeatScheduler"
+
+timezone = "UTC"
+
+task_acks_late = True
+task_reject_on_worker_lost = True
+worker_prefetch_multiplier = 1
+task_acks_on_failure_or_timeout = False
+worker_concurrency = 1

lecrapaud/jobs/scheduler.py

@@ -0,0 +1,36 @@
+from redbeat.schedulers import RedBeatSchedulerEntry
+from celery.schedules import crontab
+from lecrapaud.jobs.tasks import app
+
+
+def schedule_tasks():
+    schedule_tasks_list = [
+        {
+            "name": "task_send_daily_emails",
+            "task": "src.jobs.tasks.task_send_daily_emails",
+            "schedule": crontab(minute=00, hour=12),
+        },
+        {
+            "name": "task_training_experiment",
+            "task": "src.jobs.tasks.task_training_experiment",
+            "schedule": crontab(minute=45, hour=00),
+        },
+    ]
+
+    for task in schedule_tasks_list:
+        entry = RedBeatSchedulerEntry(**task, app=app)
+        entry.save()
+
+
+def unschedule_tasks():
+    unschedule_task_keys = [
+        "redbeat:task_send_daily_emails",
+        "redbeat:task_train_models",
+    ]
+
+    for key in unschedule_task_keys:
+        try:
+            entry = RedBeatSchedulerEntry.from_key(key, app=app)
+            entry.delete()
+        except KeyError:
+            pass

lecrapaud/jobs/tasks.py

@@ -0,0 +1,57 @@
+from lecrapaud.jobs import app
+
+# from honeybadger import honeybadger
+from lecrapaud.send_daily_emails import send_daily_emails
+from lecrapaud.config import DATASET_ID, RECEIVER_EMAIL
+from lecrapaud.training import run_training
+from lecrapaud.constants import stock_list_3
+from lecrapaud.search_space import get_models_idx
+
+
+@app.task(
+    bind=True,
+    autoretry_for=(Exception,),
+    retry_backoff=True,
+    retry_kwargs={"max_retries": 5},
+    acks_late=True,
+)
+def task_send_daily_emails(self):
+    try:
+        print(f"[Attempt #{self.request.retries}] task_send_daily_emails")
+        dataset_id = int(DATASET_ID)
+        email = RECEIVER_EMAIL
+        return send_daily_emails(email, dataset_id)
+    except Exception as e:
+        print(e)
+        # honeybadger.notify(e)
+        raise
+
+
+@app.task(
+    bind=True,
+    autoretry_for=(Exception,),
+    retry_backoff=True,
+    retry_kwargs={"max_retries": 5},
+    acks_late=True,
+)
+def task_training_experiment(self):
+    try:
+        print(f"[Attempt #{self.request.retries}] task_training_experiment")
+        run_training(
+            years_of_data=20,
+            list_of_groups=stock_list_3,
+            targets_numbers=range(1, 15),
+            percentile=20,
+            corr_threshold=80,
+            max_features=25,
+            models_idx=get_models_idx("linear", "xgb"),
+            number_of_trials=20,
+            perform_hyperoptimization=True,
+            perform_crossval=False,
+            preserve_model=False,
+            session_name="20y_stock_list_3_linear_xgb",
+        )
+    except Exception as e:
+        print(e)
+        # honeybadger.notify(e)
+        raise