CLTK Word Tokenization (Latin NLP with Python 11)
The Code:
from cltk.stem.latin.j_v import JVReplacer
from cltk.stem.latin.declension import CollatinusDecliner
from cltk.lemmatize.latin.backoff import BackoffLatinLemmatizer
from cltk.prosody.latin.macronizer import Macronizer
from cltk.tokenize.line import LineTokenizer
from cltk.tokenize.latin.sentence import SentenceTokenizer
from cltk.corpus.utils.formatter import remove_non_latin
from cltk.tokenize.word import WordTokenizer
import re
with open ('data/pl.txt', "r") as f:
text = f.read()
def jvtext(text):
j = JVReplacer()
text = j.replace(text)
return (text)
def clean_pl(text, lower=False):
cleaned = re.sub(r"[\(\[].*?[\)\]]", "", text)
cleaned = cleaned.replace(" ", " ").replace(" ", " ")
if lower==True:
lower_cleaned = cleaned.lower()
return (cleaned, lower_cleaned)
return(cleaned)
def dec(words):
decliner = CollatinusDecliner()
dec_words = {}
try:
for word in words:
dec_word = decliner.decline(word)
dec_words[word] = dec_word
except:
Exception
return (dec_words)
def lemma(tokens):
lemmatizer = BackoffLatinLemmatizer()
tokens = lemmatizer.lemmatize(tokens)
return (tokens)
def macron(text):
macronizer = Macronizer("tag_ngram_123_backoff")
text = macronizer.macronize_text(text)
return (text)
def line_tok(text):
tokenizer = LineTokenizer("latin")
text = tokenizer.tokenize(text)
return (text)
def sent_tok(text, punct=True):
sent_tokenizer = SentenceTokenizer()
sentences = sent_tokenizer.tokenize(text)
new_sents = []
if punct==True:
for sent in sentences:
sent = remove_non_latin(sent)
sent = sent.lower()
new_sents.append(sent)
return(new_sents)
return (sentences)
def word_tok(text):
word_tokenizer = WordTokenizer("latin")
words = word_tokenizer.tokenize(text)
return (words)
text = clean_pl(text)
text = jvtext(text)
sentences = sent_tok(text, punct=True)
sentence = sentences[0]
words = word_tok(sentence)
words = lemma(words)
new = []
for word in words:
new.append(word[1])
decs = dec(new)
If you enjoy this video, please subscribe. I provide all my content at no cost. If you want to support my channel, please donate via
PayPal: https://www.paypal.com/cgi-bin/webscr...
Patreon: / wjbmattingly (its my www.themedievalworld.com account as well).
If there's a specific video you would like to see or a tutorial series, let me know in the comments and I will try and make it.
If you liked this video, check out www.PythonHumanities.com, where I have Coding Exercises, Lessons, on-site Python shells where you can experiment with code, and a text version of the material discussed here.
You can follow me at:
/ wjb_mattingly
Auf dieser Seite können Sie das Online-Video CLTK Word Tokenization (Latin NLP with Python 11) mit der Dauer stunde minuten sekunde in guter Qualität ansehen, das der Benutzer Python Tutorials for Digital Humanities 04 September 2020 hochgeladen hat, den Link mit Freunden und Bekannten teilen, dieses Video wurde auf Youtube bereits 569 Mal angesehen und es wurde von 8 den Zuschauern gefallen. Viel Spaß beim Betrachtenden Zuschauern gefallen!
