indicate 0.2.0__tar.gz → 0.3.0__tar.gz

{indicate-0.2.0 → indicate-0.3.0}/PKG-INFO

@@ -1,46 +1,50 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: indicate
-Version: 0.2.0
+Version: 0.3.0
 Summary: Transliterations to/from Indian languages
-Home-page: https://github.com/in-rolls/indicate
-Author: Rajashekar Chintalapati, Gaurav Sood
-Author-email: rajshekar.ch@gmail.com, gsood07@gmail.com
+Author-email: Rajashekar Chintalapati <rajshekar.ch@gmail.com>, Gaurav Sood <gsood07@gmail.com>
 License: MIT
-Keywords: transliterate indic hindi english
+Project-URL: Homepage, https://github.com/in-rolls/indicate
+Project-URL: Repository, https://github.com/in-rolls/indicate
+Keywords: transliterate,indic,hindi,english
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Utilities
+Requires-Python: >=3.8
+Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: tqdm
 Requires-Dist: func-timeout
-Requires-Dist: tensorflow==2.18.0
+Requires-Dist: tensorflow<3.0.0,>=2.18.0
+Requires-Dist: importlib-resources>=5.0.0; python_version < "3.9"
 Provides-Extra: dev
+Requires-Dist: black>=23.0.0; extra == "dev"
+Requires-Dist: isort>=5.12.0; extra == "dev"
+Requires-Dist: flake8>=6.0.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Requires-Dist: pre-commit>=3.0.0; extra == "dev"
 Requires-Dist: check-manifest; extra == "dev"
 Provides-Extra: test
 Requires-Dist: coverage; extra == "test"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: summary
+Requires-Dist: tox; extra == "test"
+Dynamic: license-file
 
 ==================================================
 Indicate: Transliterate Indic Languages to English
 ==================================================
 
-.. image:: https://app.travis-ci.com/in-rolls/indicate.svg?branch=master
-    :target: https://travis-ci.org/in-rolls/indicate
+.. image:: https://notarypy.soodoku.workers.dev/badge/indicate/0.2.1/indicate-0.2.1-py3-none-any.whl
+    :target: https://pypi.org/integrity/indicate/0.2.1/indicate-0.2.1-py3-none-any.whl/provenance
 .. image:: https://img.shields.io/pypi/v/indicate.svg
     :target: https://pypi.python.org/pypi/indicate
 .. image:: https://readthedocs.org/projects/indicate/badge/?version=latest

{indicate-0.2.0 → indicate-0.3.0}/README.rst

@@ -2,8 +2,8 @@
 Indicate: Transliterate Indic Languages to English
 ==================================================
 
-.. image:: https://app.travis-ci.com/in-rolls/indicate.svg?branch=master
-    :target: https://travis-ci.org/in-rolls/indicate
+.. image:: https://notarypy.soodoku.workers.dev/badge/indicate/0.2.1/indicate-0.2.1-py3-none-any.whl
+    :target: https://pypi.org/integrity/indicate/0.2.1/indicate-0.2.1-py3-none-any.whl/provenance
 .. image:: https://img.shields.io/pypi/v/indicate.svg
     :target: https://pypi.python.org/pypi/indicate
 .. image:: https://readthedocs.org/projects/indicate/badge/?version=latest

{indicate-0.2.0 → indicate-0.3.0}/docs/source/conf.py

@@ -14,7 +14,7 @@ sys.path.insert(1, os.path.abspath('../../'))
 project = 'indicate'
 copyright = '2023, rajashekar chintalapati and gaurav sood'
 author = 'rajashekar chintalapati and gaurav sood'
-release = '0.1.0'
+release = '0.3.0'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

{indicate-0.2.0 → indicate-0.3.0}/indicate/data/get_affidavits/crawler/middlewares.py

@@ -3,10 +3,9 @@
 # See documentation in:
 # https://docs.scrapy.org/en/latest/topics/spider-middleware.html
 
-from scrapy import signals
-
 # useful for handling different item types with a single interface
-from itemadapter import is_item, ItemAdapter
+from itemadapter import ItemAdapter, is_item
+from scrapy import signals
 
 
 class CrawlerSpiderMiddleware:
@@ -53,7 +52,7 @@ class CrawlerSpiderMiddleware:
             yield r
 
     def spider_opened(self, spider):
-        spider.logger.info('Spider opened: %s' % spider.name)
+        spider.logger.info("Spider opened: %s" % spider.name)
 
 
 class CrawlerDownloaderMiddleware:
@@ -100,4 +99,4 @@ class CrawlerDownloaderMiddleware:
         pass
 
     def spider_opened(self, spider):
-        spider.logger.info('Spider opened: %s' % spider.name)
+        spider.logger.info("Spider opened: %s" % spider.name)

{indicate-0.2.0 → indicate-0.3.0}/indicate/data/get_affidavits/crawler/settings.py

@@ -7,82 +7,82 @@
 #     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
 #     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
 
-BOT_NAME = 'crawler'
+BOT_NAME = "crawler"
 
-SPIDER_MODULES = ['crawler.spiders']
-NEWSPIDER_MODULE = 'crawler.spiders'
+SPIDER_MODULES = ["crawler.spiders"]
+NEWSPIDER_MODULE = "crawler.spiders"
 
 
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
-USER_AGENT = 'crawler (+https://affidavit.eci.gov.in)'
+USER_AGENT = "crawler (+https://affidavit.eci.gov.in)"
 
 # Obey robots.txt rules
 ROBOTSTXT_OBEY = True
 
 # Configure maximum concurrent requests performed by Scrapy (default: 16)
-#CONCURRENT_REQUESTS = 32
+# CONCURRENT_REQUESTS = 32
 
 # Configure a delay for requests for the same website (default: 0)
 # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
+# DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
-#CONCURRENT_REQUESTS_PER_DOMAIN = 16
-#CONCURRENT_REQUESTS_PER_IP = 16
+# CONCURRENT_REQUESTS_PER_DOMAIN = 16
+# CONCURRENT_REQUESTS_PER_IP = 16
 
 # Disable cookies (enabled by default)
-#COOKIES_ENABLED = False
+# COOKIES_ENABLED = False
 
 # Disable Telnet Console (enabled by default)
-#TELNETCONSOLE_ENABLED = False
+# TELNETCONSOLE_ENABLED = False
 
 # Override the default request headers:
-#DEFAULT_REQUEST_HEADERS = {
+# DEFAULT_REQUEST_HEADERS = {
 #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 #   'Accept-Language': 'en',
-#}
+# }
 
 # Enable or disable spider middlewares
 # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
-#SPIDER_MIDDLEWARES = {
+# SPIDER_MIDDLEWARES = {
 #    'crawler.middlewares.CrawlerSpiderMiddleware': 543,
-#}
+# }
 
 # Enable or disable downloader middlewares
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
-#DOWNLOADER_MIDDLEWARES = {
+# DOWNLOADER_MIDDLEWARES = {
 #    'crawler.middlewares.CrawlerDownloaderMiddleware': 543,
-#}
+# }
 
 # Enable or disable extensions
 # See https://docs.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
+# EXTENSIONS = {
 #    'scrapy.extensions.telnet.TelnetConsole': None,
-#}
+# }
 
 # Configure item pipelines
 # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
-#ITEM_PIPELINES = {
+# ITEM_PIPELINES = {
 #    'crawler.pipelines.CrawlerPipeline': 300,
-#}
+# }
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
+# AUTOTHROTTLE_ENABLED = True
 # The initial download delay
-#AUTOTHROTTLE_START_DELAY = 5
+# AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
-#AUTOTHROTTLE_MAX_DELAY = 60
+# AUTOTHROTTLE_MAX_DELAY = 60
 # The average number of requests Scrapy should be sending in parallel to
 # each remote server
-#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 # Enable showing throttling stats for every response received:
-#AUTOTHROTTLE_DEBUG = False
+# AUTOTHROTTLE_DEBUG = False
 
 # Enable and configure HTTP caching (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
-#HTTPCACHE_DIR = 'httpcache'
-#HTTPCACHE_IGNORE_HTTP_CODES = []
-#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+# HTTPCACHE_ENABLED = True
+# HTTPCACHE_EXPIRATION_SECS = 0
+# HTTPCACHE_DIR = 'httpcache'
+# HTTPCACHE_IGNORE_HTTP_CODES = []
+# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

indicate-0.3.0/indicate/data/get_affidavits/crawler/spiders/crawler_spider.py

@@ -0,0 +1,118 @@
+import re
+
+from crawler.items import CrawlerItem, Fixture
+from scrapy import Request, Spider
+from scrapy.linkextractors import LinkExtractor
+from scrapy.selector import Selector
+from scrapy.spiders import CrawlSpider, Rule
+
+
+class CrawlerSpider(Spider):
+    name = "crawler"
+    allowed_domains = ["affidavit.eci.gov.in"]
+    start_urls = [
+        "https://affidavit.eci.gov.in",
+    ]
+    rules = [
+        Rule(LinkExtractor(allow=r"show-profile"), callback="parse_product"),
+    ]
+    unique_id = 0
+
+    def parse(self, response):
+        # questions = Selector(response).xpath('//ul[@class="table"]/li')
+        questions = response.selector
+        print("11111111111111111111", questions)
+        # get next page
+        for p in questions.css(".pagination"):
+            detail_link = p.xpath('li[@class="page-item"]')
+            # print ('------------------', detail_link)
+            for link in detail_link:
+                n = link.extract()
+                # print ('_____________________',n)
+                matches = re.findall('rel="next"', n)
+                # print(matches)
+                if matches:
+                    next_link = link.xpath("a/@href").extract()
+                    print("kakakakakakak", next_link)
+                    yield Request(next_link[0], callback=self.parse)
+
+        # item['name_english'] = tr.xpath('td[2]/div[@class="details-name"]/h4/text()').extract()
+
+        for tr in questions.css("table#data-tab>tbody>tr"):
+            item = Fixture()
+            detail_link = tr.xpath('td[1]/div[@class="img-bx"]/a/@href').extract()
+            # print ('=======================', detail_link)
+            item["name_english"] = tr.xpath(
+                'td[2]/div[@class="details-name"]/h4/text()'
+            ).extract()
+            # item['link'] =detail_link
+            # yield item
+            yield Request(detail_link[0], callback=self.parse_product)
+
+    def parse_product(self, response):
+        # questions = Selector(response).xpath('//ul[@class="table"]/li')
+        questions = response.selector
+        print("2222222222222222", questions)
+        # for sec in questions.css('section'):
+        #     print ('3333333333333333',sec)
+        can_details = questions.css("section")[0]
+        # can_per_details = questions.css('section')[1]
+        can_details = can_details.xpath(
+            'div/div/div/div/div[@class="card-body"]/div/div'
+        )
+        can_details = can_details[1].xpath("div")
+        # name = can_details[3]
+
+        item = Fixture()
+        self.unique_id += 1
+        item["unique_id"] = self.unique_id
+        item["party_name"] = (
+            can_details[0].xpath("div[2]/text()").extract()[0].strip("\n").strip()
+        )
+        item["name_english"] = (
+            can_details[2]
+            .xpath("div[2]/div/p/b/text()")
+            .extract()[0]
+            .strip("\n")
+            .strip()
+        )
+        item["name_hindi"] = (
+            can_details[3]
+            .xpath("div[2]/div/p/b/text()")
+            .extract()[0]
+            .strip("\n")
+            .strip()
+        )
+        item["assembly_constituency"] = (
+            can_details[4].xpath("div[2]/div/p/text()").extract()[0].strip("\n").strip()
+        )
+        item["state"] = (
+            can_details[5].xpath("div[2]/div/p/text()").extract()[0].strip("\n").strip()
+        )
+        item["application_uploaded"] = (
+            can_details[6].xpath("div[2]/div/p/text()").extract()
+        )
+        item["current_status"] = can_details[7].xpath("div[2]/div/p/text()").extract()
+
+        can_per_details = questions.css("section")[1]
+        can_per_details = can_per_details.xpath(
+            'div/div[@class="row"]/div/div/div/div/div/form/div/div[1]/div'
+        )
+
+        item["fathers_or_husbands_name_english"] = (
+            can_per_details[0].xpath("div/p/text()").extract()
+        )
+        item["fathers_or_husbands_name_hindi"] = (
+            can_per_details[1].xpath("div/p/text()").extract()
+        )
+        item["name"] = can_per_details[2].xpath("div/p/text()").extract()
+
+        item["address"] = can_per_details[4].xpath("div/p/text()").extract()
+        item["age"] = can_per_details[6].xpath("div/p/text()").extract()
+        # item['url'] = can_per_details[0].xpath('div[2]/text()').extract()[0].strip('\n').strip()
+        # item['photo_url'] = can_per_details[0].xpath('div[2]/text()').extract()[0].strip('\n').strip()
+        # item['photo_filename_with_unique_id'] = can_per_details[0].xpath('div[2]/text()').extract()[0].strip('\n').strip()
+        # item['affidavit_filename_with_unique_id'] = can_per_details[0].xpath('div[2]/text()').extract()[0].strip(
+        #'\n').strip()
+
+        yield item

{indicate-0.2.0 → indicate-0.3.0}/indicate/data/get_espncricinfo/find_hindi_scorecards.py

@@ -1,5 +1,6 @@
-import json
 import datetime
+import json
+
 import requests
 from espncricinfo.match import Match
 
@@ -19,7 +20,9 @@ with open("all-matches.json", "r") as matches_file:
         if datetime.datetime(y, m, d) > datetime.datetime(2021, 4, 8):
             print(match_id)
             r = requests.get(match.match_url)
-            hindi_url = r.url.replace("/series/", "/hindi/series/").replace("live-cricket-score", "full-scorecard")
+            hindi_url = r.url.replace("/series/", "/hindi/series/").replace(
+                "live-cricket-score", "full-scorecard"
+            )
             r = requests.get(hindi_url)
             if r.status_code == 404:
                 hindi_url = None

indicate-0.3.0/indicate/data/get_espncricinfo/parse_hindi_scorecards.py

@@ -0,0 +1,42 @@
+import datetime
+import json
+
+import requests
+from bs4 import BeautifulSoup
+from espncricinfo.match import Match
+
+players = []
+
+with open("hindi_matches.json", "r") as matches_file:
+    hindi_matches = json.load(matches_file)
+    for match in hindi_matches:
+        print(match)
+        m = Match(match)
+        r = requests.get(m.match_url)
+        hindi_url = r.url.replace("/series/", "/hindi/series/").replace(
+            "live-cricket-score", "full-scorecard"
+        )
+        hindi_content = requests.get(hindi_url).text
+        soup = BeautifulSoup(hindi_content, "html.parser")
+        text = soup.find("script", id="__NEXT_DATA__").string
+        hindi_json = json.loads(text)
+        if "data" in hindi_json["props"]["pageProps"]:
+            for team_players in hindi_json["props"]["pageProps"]["data"]["pageData"][
+                "content"
+            ]["matchPlayers"]["teamPlayers"]:
+                for player in team_players["players"]:
+                    players.append(
+                        {
+                            "id": player["player"]["id"],
+                            "hindi_name": player["player"]["name"],
+                            "hindi_long_name": player["player"]["longName"],
+                            "english_name": player["player"]["indexName"],
+                            "gender": player["player"]["gender"],
+                            "date_of_birth": player["player"]["dateOfBirth"],
+                            "country_id": player["player"]["countryTeamId"],
+                            "slug": player["player"]["slug"],
+                        }
+                    )
+
+with open("players_with_hindi_names.json", "w") as outfile:
+    json.dump(players, outfile)

{indicate-0.2.0 → indicate-0.3.0}/indicate/decoder.py

@@ -1,12 +1,23 @@
+from typing import Any, List, Optional, Tuple
+
 import tensorflow as tf
 
 from .logging import get_logger
 
 logger = get_logger()
 
+
 class Decoder(tf.keras.Model):
-    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz,
-                 max_length_input, max_length_output, attention_type='luong'):
+    def __init__(
+        self,
+        vocab_size: int,
+        embedding_dim: int,
+        dec_units: int,
+        batch_sz: int,
+        max_length_input: int,
+        max_length_output: int,
+        attention_type: str = "luong",
+    ) -> None:
         super(Decoder, self).__init__()
         self.batch_sz = batch_sz
         self.dec_units = dec_units
@@ -22,32 +33,43 @@ class Decoder(tf.keras.Model):
 
         # Define the fundamental cell for decoder recurrent structure
         self.decoder_rnn_cell = tf.keras.layers.LSTMCell(self.dec_units)
-        self.decoder_rnn = tf.keras.layers.RNN(self.decoder_rnn_cell, return_sequences=True, return_state=True)
+        self.decoder_rnn = tf.keras.layers.RNN(
+            self.decoder_rnn_cell, return_sequences=True, return_state=True
+        )
 
         # Create attention mechanism
         # For Luong attention: project the query to match the encoder output's dimension (dec_units)
-        if self.attention_type == 'luong':
+        if self.attention_type == "luong":
             self.query_layer = tf.keras.layers.Dense(dec_units)
 
         self.attention_mechanism = self.build_attention_mechanism(None)
 
-    def build_attention_mechanism(self, memory):
-        if self.attention_type == 'bahdanau':
+    def build_attention_mechanism(
+        self, memory: Optional[tf.Tensor]
+    ) -> tf.keras.layers.Layer:
+        if self.attention_type == "bahdanau":
             return tf.keras.layers.AdditiveAttention()
         else:
             return tf.keras.layers.Attention()
 
-    def call(self, inputs, initial_state, encoder_outputs):
+    def call(
+        self,
+        inputs: tf.Tensor,
+        initial_state: List[tf.Tensor],
+        encoder_outputs: tf.Tensor,
+    ) -> Tuple[tf.Tensor, Tuple[tf.Tensor, tf.Tensor], tf.Tensor]:
         # Get the embeddings of the inputs
         x = self.embedding(inputs)
 
-        if self.attention_type == 'luong':
-          query = self.query_layer(x)
+        if self.attention_type == "luong":
+            query = self.query_layer(x)
         else:
-          query = x
+            query = x
 
         # Feature dimensions to the attention layer
-        attention_output, attention_weights = self.attention_mechanism([query, encoder_outputs], return_attention_scores=True)
+        attention_output, attention_weights = self.attention_mechanism(
+            [query, encoder_outputs], return_attention_scores=True
+        )
         # Concatenate the attention output with the LSTM Cell output
         lstm_input = tf.concat([attention_output, x], axis=-1)
         # Process through the LSTM cell
@@ -56,7 +78,8 @@ class Decoder(tf.keras.Model):
         outputs = self.fc(outputs)
         return outputs, (state_h, state_c), attention_weights
 
-    def build_initial_state(self, batch_sz, encoder_state):
+    def build_initial_state(
+        self, batch_sz: int, encoder_state: List[tf.Tensor]
+    ) -> List[tf.Tensor]:
         hidden_state, cell_state = encoder_state
         return [hidden_state, cell_state]
-    

indicate-0.3.0/indicate/encoder.py

@@ -0,0 +1,34 @@
+from typing import Any, List, Tuple
+
+import tensorflow as tf
+
+
+class Encoder(tf.keras.Model):
+    def __init__(
+        self, vocab_size: int, embedding_dim: int, enc_units: int, batch_sz: int
+    ) -> None:
+        super(Encoder, self).__init__()
+        self.batch_sz = batch_sz
+        self.enc_units = enc_units
+        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)
+
+        ##-------- LSTM layer in Encoder ------- ##
+        self.lstm_layer = tf.keras.layers.LSTM(
+            self.enc_units,
+            return_sequences=True,
+            return_state=True,
+            recurrent_initializer="glorot_uniform",
+        )
+
+    def call(
+        self, x: tf.Tensor, hidden: List[tf.Tensor], training: bool = None
+    ) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor]:
+        x = self.embedding(x)
+        output, h, c = self.lstm_layer(x, initial_state=hidden)
+        return output, h, c
+
+    def initialize_hidden_state(self) -> List[tf.Tensor]:
+        return [
+            tf.zeros((self.batch_sz, self.enc_units)),
+            tf.zeros((self.batch_sz, self.enc_units)),
+        ]