+
    ÖÇÜhá  ã                   óÂ   € R t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RIH	t	 ]R 4       t
]
P                  ]4      R 4       t]
P                  ]4      R 4       t ! R	 R
4      tR# )zLanguage Model VocabularyN)ÚCounter)ÚIterable)Úsingledispatch)Úchainc                 ó0   € \        R \        V 4       24      h)z/Unsupported type for looking up in vocabulary: )Ú	TypeErrorÚtype©ÚwordsÚvocabs   &&Ú4/usr/lib/python3/dist-packages/nltk/lm/vocabulary.pyÚ_dispatched_lookupr      s   € ä
ÐEÄdÈ5ÃkÀ]ÐSÓ
TÐTó    c                óh   a€ \         ;QJ d    . V3R lV  4       F  NK  	  5# ! V3R lV  4       4      # )z[Look up a sequence of words in the vocabulary.

Returns an iterator over looked up words.

c              3   ó<   <"  € T F  p\        VS4      x € K  	  R # 5i©N©r   )Ú.0Úwr   s   & €r   Ú	<genexpr>Ú_.<locals>.<genexpr>   s   øé € Ð=±u°!Ô# A u×-Ð-³uùs   ƒ)Útupler	   s   &fr   Ú_r      s'   ø€ ÷ Œ5Ô=±uÓ=�5Ð=ˆ5Ô=±uÓ=Ó=Ð=r   c                ó*   € W9   d   V # VP                   # )z$Looks up one word in the vocabulary.)Ú	unk_label)Úwordr   s   &&r   Ú_string_lookupr      s   € ð ”=ˆ4Ð5 e§o¡oÐ5r   c                   ón   a € ] tR t^%t o RtRR lt]R 4       tR tR t	R t
R tR	 tR
 tR tR tRtV tR# )Ú
Vocabularya¸	  Stores language model vocabulary.

Satisfies two common language modeling requirements for a vocabulary:

- When checking membership and calculating its size, filters items
  by comparing their counts to a cutoff value.
- Adds a special "unknown" token which unseen words are mapped to.

>>> words = ['a', 'c', '-', 'd', 'c', 'a', 'b', 'r', 'a', 'c', 'd']
>>> from nltk.lm import Vocabulary
>>> vocab = Vocabulary(words, unk_cutoff=2)

Tokens with counts greater than or equal to the cutoff value will
be considered part of the vocabulary.

>>> vocab['c']
3
>>> 'c' in vocab
True
>>> vocab['d']
2
>>> 'd' in vocab
True

Tokens with frequency counts less than the cutoff value will be considered not
part of the vocabulary even though their entries in the count dictionary are
preserved.

>>> vocab['b']
1
>>> 'b' in vocab
False
>>> vocab['aliens']
0
>>> 'aliens' in vocab
False

Keeping the count entries for seen words allows us to change the cutoff value
without having to recalculate the counts.

>>> vocab2 = Vocabulary(vocab.counts, unk_cutoff=1)
>>> "b" in vocab2
True

The cutoff value influences not only membership checking but also the result of
getting the size of the vocabulary using the built-in `len`.
Note that while the number of keys in the vocabulary's counter stays the same,
the items in the vocabulary differ depending on the cutoff.
We use `sorted` to demonstrate because it keeps the order consistent.

>>> sorted(vocab2.counts)
['-', 'a', 'b', 'c', 'd', 'r']
>>> sorted(vocab2)
['-', '<UNK>', 'a', 'b', 'c', 'd', 'r']
>>> sorted(vocab.counts)
['-', 'a', 'b', 'c', 'd', 'r']
>>> sorted(vocab)
['<UNK>', 'a', 'c', 'd']

In addition to items it gets populated with, the vocabulary stores a special
token that stands in for so-called "unknown" items. By default it's "<UNK>".

>>> "<UNK>" in vocab
True

We can look up words in a vocabulary using its `lookup` method.
"Unseen" words (with counts less than cutoff) are looked up as the unknown label.
If given one word (a string) as an input, this method will return a string.

>>> vocab.lookup("a")
'a'
>>> vocab.lookup("aliens")
'<UNK>'

If given a sequence, it will return an tuple of the looked up words.

>>> vocab.lookup(["p", 'a', 'r', 'd', 'b', 'c'])
('<UNK>', 'a', '<UNK>', 'd', '<UNK>', 'c')

It's possible to update the counts after the vocabulary has been created.
In general, the interface is the same as that of `collections.Counter`.

>>> vocab['b']
1
>>> vocab.update(["b", "b", "c"])
>>> vocab['b']
3
Nc                ó    € W0n         V^8  d   \        RV 24      hW n        \        4       V n        T P                  Ve	   V4       R# R4       R# )a“  Create a new Vocabulary.

:param counts: Optional iterable or `collections.Counter` instance to
               pre-seed the Vocabulary. In case it is iterable, counts
               are calculated.
:param int unk_cutoff: Words that occur less frequently than this value
                       are not considered part of the vocabulary.
:param unk_label: Label for marking words not part of vocabulary.

z)Cutoff value cannot be less than 1. Got: NÚ )r   Ú
ValueErrorÚ_cutoffr   ÚcountsÚupdate)Úselfr#   Ú
unk_cutoffr   s   &&&&r   Ú__init__ÚVocabulary.__init__   sG   € ð #ŒØ˜Œ>ÜÐHÈÈÐUÓVÐVØ!Œä“iˆŒØ�‰˜fÒ0�FÖ9°bÖ9r   c                ó   € V P                   # )zYCutoff value.

Items with count below this value are not considered part of vocabulary.

)r"   ©r%   s   &r   ÚcutoffÚVocabulary.cutoff’   s   € ð �|‰|Ðr   c                ól   € V P                   P                  ! V/ VB  \        R V  4       4      V n        R# )zGUpdate vocabulary counts.

Wraps `collections.Counter.update` method.

c              3   ó&   "  € T F  p^x € K	  	  R# 5i)é   N© )r   r   s   & r   r   Ú$Vocabulary.update.<locals>.<genexpr>¢   s   é € Ð(¡4˜a�£4ùs   ‚N)r#   r$   ÚsumÚ_len)r%   Úcounter_argsÚcounter_kwargss   &*,r   r$   ÚVocabulary.update›   s-   € ð 	�‰×Ò˜LÐ;¨NÒ;ÜÑ(¡4Ó(Ó(ˆŽ	r   c                ó   € \        W4      # )a—  Look up one or more words in the vocabulary.

If passed one word as a string will return that word or `self.unk_label`.
Otherwise will assume it was passed a sequence of words, will try to look
each of them up and return an iterator over the looked up words.

:param words: Word(s) to look up.
:type words: Iterable(str) or str
:rtype: generator(str) or str
:raises: TypeError for types other than strings or iterables

>>> from nltk.lm import Vocabulary
>>> vocab = Vocabulary(["a", "b", "c", "a", "b"], unk_cutoff=2)
>>> vocab.lookup("a")
'a'
>>> vocab.lookup("aliens")
'<UNK>'
>>> vocab.lookup(["a", "b", "c", ["x", "b"]])
('a', 'b', '<UNK>', ('<UNK>', 'b'))

r   )r%   r
   s   &&r   ÚlookupÚVocabulary.lookup¤   s   € ô, " %Ó.Ð.r   c                ó`   € WP                   8X  d   V P                  # V P                  V,          # r   )r   r"   r#   ©r%   Úitems   &&r   Ú__getitem__ÚVocabulary.__getitem__¼   s#   € Ø#§~¡~Ô5ˆt�|‰|ÐL¸4¿;¹;ÀtÕ;LÐLr   c                ó,   € W,          V P                   8¬  # )zHOnly consider items with counts GE to cutoff as being in the
vocabulary.)r+   r;   s   &&r   Ú__contains__ÚVocabulary.__contains__¿   s   € ð �z˜TŸ[™[Ñ(Ð(r   c                óŠ   a € \        V 3R lS P                   4       S P                  '       d   S P                  .4      # . 4      # )zCBuilding on membership check define how to iterate over
vocabulary.c              3   ó8   <"  € T F  qS9   g   K  Vx € K  	  R # 5ir   r0   )r   r<   r%   s   & €r   r   Ú&Vocabulary.__iter__.<locals>.<genexpr>È   s   øé € Ð:™k�d°T©\�TŠT›kùs   ƒ�
)r   r#   r   r*   s   fr   Ú__iter__ÚVocabulary.__iter__Ä   s=   ø€ ô Ü:˜dŸkškÓ:Ø $§§ ˆT�^‰^Ðó
ð 	
à13ó
ð 	
r   c                ó   € V P                   # )z1Computing size of vocabulary reflects the cutoff.)r3   r*   s   &r   Ú__len__ÚVocabulary.__len__Ì   s   € à�y‰yÐr   c                ó¸   € V P                   VP                   8H  ;'       d;    V P                  VP                  8H  ;'       d    V P                  VP                  8H  # r   )r   r+   r#   )r%   Úothers   &&r   Ú__eq__ÚVocabulary.__eq__Ð   sK   € à�N‰N˜eŸo™oÑ-÷ ,ð ,Ø—‘˜uŸ|™|Ñ+÷,ð ,à—‘˜uŸ|™|Ñ+ð	
r   c                óŒ   € R P                  V P                  P                  V P                  V P                  \        V 4      4      # )z/<{} with cutoff={} unk_label='{}' and {} items>)ÚformatÚ	__class__Ú__name__r+   r   Úlenr*   s   &r   Ú__str__ÚVocabulary.__str__×   s4   € Ø@×GÑGØ�N‰N×#Ñ# T§[¡[°$·.±.Ä#ÀdÃ)ó
ð 	
r   )r"   r3   r#   r   )Nr/   z<UNK>)rQ   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r'   Úpropertyr+   r$   r8   r=   r@   rE   rH   rL   rS   Ú__static_attributes__Ú__classdictcell__)Ú__classdict__s   @r   r   r   %   sS   ø‡ € ñWôr:ð& ñó ðò)ò/ò0Mò)ò

òò
÷
ð 
r   r   )rX   ÚsysÚcollectionsr   Úcollections.abcr   Ú	functoolsr   Ú	itertoolsr   r   Úregisterr   Ústrr   r   r0   r   r   Ú<module>rd      sy   ðñ  ã 
Ý Ý $Ý $Ý ð ñUó ðUð ×Ñ˜XÓ&ñ>ó 'ð>ð ×Ñ˜SÓ!ñ6ó "ð6÷
u
ó u
r   