PgsFile 0.3.8__py3-none-any.whl → 0.3.9__py3-none-any.whl

PgsFile/Corpora/Stopwords/english.txt

@@ -1,11 +1,10 @@
+'d
 'll
-'tis
-'twas
+'m
+'re
+'s
 've
-10
-39
 a
-a's
 able
 ableabout
 about
@@ -36,8 +35,8 @@ ago
 ah
 ahead
 ai
-ain't
 aint
+ain't
 al
 all
 allow
@@ -83,12 +82,13 @@ are
 area
 areas
 aren
-aren't
 arent
+aren't
 arise
 around
 arpa
 as
+a's
 aside
 ask
 asked
@@ -164,15 +164,13 @@ bw
 by
 bz
 c
-c'mon
-c's
 ca
 call
 came
 can
-can't
 cannot
 cant
+can't
 caption
 case
 cases
@@ -194,6 +192,7 @@ clearly
 click
 cm
 cmon
+c'mon
 cn
 co
 co.
@@ -212,14 +211,15 @@ contains
 copy
 corresponding
 could
-could've
 couldn
-couldn't
 couldnt
+couldn't
+could've
 course
 cr
 cry
 cs
+c's
 cu
 currently
 cv
@@ -227,9 +227,10 @@ cx
 cy
 cz
 d
+d
 dare
-daren't
 darent
+daren't
 date
 de
 dear
@@ -240,8 +241,8 @@ despite
 detail
 did
 didn
-didn't
 didnt
+didn't
 differ
 different
 differently
@@ -252,13 +253,13 @@ dm
 do
 does
 doesn
-doesn't
 doesnt
+doesn't
 doing
 don
-don't
 done
 dont
+don't
 doubtful
 down
 downed
@@ -412,41 +413,40 @@ gw
 gy
 h
 had
-hadn't
 hadnt
+hadn't
 half
 happens
 hardly
 has
 hasn
-hasn't
 hasnt
+hasn't
 have
 haven
-haven't
 havent
+haven't
 having
 he
-he'd
-he'll
-he's
 hed
+he'd
 hell
+he'll
 hello
 help
 hence
 her
 here
-here's
 hereafter
 hereby
 herein
 heres
+here's
 hereupon
 hers
 herself
-herse”
 hes
+he's
 hi
 hid
 high
@@ -454,7 +454,6 @@ higher
 highest
 him
 himself
-himse”
 his
 hither
 hk
@@ -464,11 +463,11 @@ home
 homepage
 hopefully
 how
+howbeit
 how'd
+however
 how'll
 how's
-howbeit
-however
 hr
 ht
 htm
@@ -477,19 +476,18 @@ http
 hu
 hundred
 i
-i'd
-i'll
-i'm
-i've
 i.e.
 id
+i'd
 ie
 if
 ignored
 ii
 il
 ill
+i'll
 im
+i'm
 immediate
 immediately
 importance
@@ -521,18 +519,18 @@ iq
 ir
 is
 isn
-isn't
 isnt
+isn't
 it
-it'd
-it'll
-it's
 itd
+it'd
 itll
+it'll
 its
+it's
 itself
-itse”
 ive
+i've
 j
 je
 jm
@@ -578,8 +576,8 @@ length
 less
 lest
 let
-let's
 lets
+let's
 li
 like
 liked
@@ -589,6 +587,7 @@ line
 little
 lk
 ll
+ll
 long
 longer
 longest
@@ -605,6 +604,7 @@ lu
 lv
 ly
 m
+m
 ma
 made
 mainly
@@ -615,8 +615,8 @@ man
 many
 may
 maybe
-mayn't
 maynt
+mayn't
 mc
 md
 me
@@ -632,9 +632,9 @@ mg
 mh
 microsoft
 might
-might've
-mightn't
 mightnt
+mightn't
+might've
 mil
 mill
 million
@@ -662,17 +662,17 @@ mu
 much
 mug
 must
-must've
-mustn't
 mustnt
+mustn't
+must've
 mv
 mw
 mx
 my
 myself
-myse”
 mz
 n
+n't
 na
 name
 namely
@@ -687,8 +687,8 @@ necessary
 need
 needed
 needing
-needn't
 neednt
+needn't
 needs
 neither
 net
@@ -708,12 +708,12 @@ nine
 ninety
 nl
 no
-no-one
 nobody
 non
 none
 nonetheless
 noone
+no-one
 nor
 normally
 nos
@@ -726,6 +726,7 @@ now
 nowhere
 np
 nr
+n't
 nu
 null
 number
@@ -749,8 +750,8 @@ omitted
 on
 once
 one
-one's
 ones
+one's
 only
 onto
 open
@@ -769,8 +770,8 @@ other
 others
 otherwise
 ought
-oughtn't
 oughtnt
+oughtn't
 our
 ours
 ourselves
@@ -848,6 +849,7 @@ ran
 rather
 rd
 re
+re
 readily
 really
 reasonably
@@ -876,6 +878,7 @@ ru
 run
 rw
 s
+s
 sa
 said
 same
@@ -912,20 +915,20 @@ several
 sg
 sh
 shall
-shan't
 shant
+shan't
 she
-she'd
-she'll
-she's
 shed
+she'd
 shell
+she'll
 shes
+she's
 should
-should've
 shouldn
-shouldn't
 shouldnt
+shouldn't
+should've
 show
 showed
 showing
@@ -992,7 +995,6 @@ sy
 system
 sz
 t
-t's
 take
 taken
 taking
@@ -1011,12 +1013,12 @@ thank
 thanks
 thanx
 that
-that'll
-that's
-that've
 thatll
+that'll
 thats
+that's
 thatve
+that've
 the
 their
 theirs
@@ -1025,33 +1027,33 @@ themselves
 then
 thence
 there
-there'd
-there'll
-there're
-there's
-there've
 thereafter
 thereby
 thered
+there'd
 therefore
 therein
 therell
+there'll
 thereof
 therere
+there're
 theres
+there's
 thereto
 thereupon
 thereve
+there've
 these
 they
-they'd
-they'll
-they're
-they've
 theyd
+they'd
 theyll
+they'll
 theyre
+they're
 theyve
+they've
 thick
 thin
 thing
@@ -1080,6 +1082,7 @@ til
 till
 tip
 tis
+tis
 tj
 tk
 tm
@@ -1101,6 +1104,7 @@ truly
 try
 trying
 ts
+t's
 tt
 turn
 turned
@@ -1109,6 +1113,7 @@ turns
 tv
 tw
 twas
+twas
 twelve
 twenty
 twice
@@ -1151,6 +1156,7 @@ value
 various
 vc
 ve
+ve
 versus
 very
 vg
@@ -1169,53 +1175,53 @@ wanting
 wants
 was
 wasn
-wasn't
 wasnt
+wasn't
 way
 ways
 we
-we'd
-we'll
-we're
-we've
 web
 webpage
 website
 wed
+we'd
 welcome
 well
+we'll
 wells
 went
 were
+we're
 weren
-weren't
 werent
+weren't
 weve
+we've
 wf
 what
 what'd
-what'll
-what's
-what've
 whatever
 whatll
+what'll
 whats
+what's
 whatve
+what've
 when
+whence
 when'd
+whenever
 when'll
 when's
-whence
-whenever
 where
-where'd
-where'll
-where's
 whereafter
 whereas
 whereby
+where'd
 wherein
+where'll
 wheres
+where's
 whereupon
 wherever
 whether
@@ -1226,16 +1232,16 @@ whilst
 whim
 whither
 who
-who'd
-who'll
-who's
 whod
+who'd
 whoever
 whole
 wholl
+who'll
 whom
 whomever
 whos
+who's
 whose
 why
 why'd
@@ -1250,9 +1256,9 @@ with
 within
 without
 won
-won't
 wonder
 wont
+won't
 words
 work
 worked
@@ -1260,10 +1266,10 @@ working
 works
 world
 would
-would've
 wouldn
-wouldn't
 wouldnt
+wouldn't
+would've
 ws
 www
 x
@@ -1274,25 +1280,25 @@ years
 yes
 yet
 you
-you'd
-you'll
-you're
-you've
 youd
+you'd
 youll
+you'll
 young
 younger
 youngest
 your
 youre
+you're
 yours
 yourself
 yourselves
 youve
+you've
 yt
 yu
 z
 za
 zero
 zm
-zr
+zr

PgsFile/PgsFile.py

@@ -3795,4 +3795,60 @@ def perform_liwc_zh(dic_path, file_path, output_excel_path):
     
     import pandas as pd
     df = pd.DataFrame(data,columns=[u'类别', u'出现词种数', u'占词表百分比', u'出现词次', u'总词次', u'覆盖率', u'例词'])
-    df.to_excel(output_excel_path,'sheet1',index=False)
+    df.to_excel(output_excel_path,'sheet1',index=False)
+    
+    
+import math
+from collections import defaultdict
+def calculate_log_likelihood(target_count, reference_count, total_target, total_reference):
+    """Calculate the log-likelihood of a word being a keyword using absolute frequencies."""
+    # Calculate expected frequencies
+    total_combined = total_target + total_reference
+    expected_target = (target_count + reference_count) * (total_target / total_combined)
+    expected_reference = (target_count + reference_count) * (total_reference / total_combined)
+
+    # Calculate log-likelihood
+    ll = 0.0
+    if target_count > 0:
+        ll += target_count * math.log(target_count / expected_target)
+    if reference_count > 0:
+        ll += reference_count * math.log(reference_count / expected_reference)
+
+    return ll * 2  # Return G^2 statistic
+
+def extract_keywords_en(target_text, top_n=10):
+    """Extract keywords from target text using log-likelihood with absolute reference frequencies."""
+    # Example usage
+    my_dic_path = get_library_location("PgsFile")+"/PgsFile/models/dics/unigram_freq_only.json"
+    reference_freq = get_data_json(my_dic_path)
+    # Tokenize target text and preserve original case
+    original_words = word_tokenize2(target_text)
+    lower_words = [w.lower() for w in original_words if w.lower() not in BigPunctuation and w.lower() not in get_stopwords()]
+    total_target = len(lower_words)
+
+    # Calculate target word frequencies
+    target_word_freq = defaultdict(int)
+    word_case_mapping = {}
+    for orig_word, lower_word in zip(original_words, [w.lower() for w in original_words]):
+        if lower_word in lower_words:
+            target_word_freq[lower_word] += 1
+            if lower_word not in word_case_mapping:
+                word_case_mapping[lower_word] = orig_word
+
+    # Calculate total reference frequency
+    total_reference = sum(reference_freq.values())
+
+    # Calculate log-likelihood for each word
+    keyword_scores = []
+    for word, target_count in target_word_freq.items():
+        reference_count = reference_freq.get(word, 0)
+        ll = calculate_log_likelihood(target_count, reference_count, total_target, total_reference)
+        relative_freq = target_count / total_target
+        original_word = word_case_mapping.get(word, word)
+        keyword_scores.append((original_word, target_count, relative_freq, ll))
+
+    # Sort keywords by log-likelihood score
+    keyword_scores.sort(key=lambda x: x[3], reverse=True)
+
+    # Return top N keywords
+    return keyword_scores[:top_n]

PgsFile/__init__.py

@@ -49,7 +49,7 @@ from .PgsFile import ngrams, bigrams, trigrams, everygrams, compute_similarity,
 from .PgsFile import word_list, batch_word_list
 from .PgsFile import cs, cs1, sent_tokenize, word_tokenize, word_tokenize2
 from .PgsFile import word_lemmatize, word_POS, word_NER
-from .PgsFile import extract_noun_phrases, get_LLMs_prompt
+from .PgsFile import extract_noun_phrases, get_LLMs_prompt, extract_keywords_en
 from .PgsFile import extract_dependency_relations, extract_dependency_relations_full
 from .PgsFile import predict_category