Mercurial > repos > bcclaywell > argo_navis

comparison venv/lib/python2.7/site-packages/requests/packages/chardet/sjisprober.py @ 0:d67268158946 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b
author bcclaywell
date Mon, 12 Oct 2015 17:43:33 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d67268158946
1 ######################## BEGIN LICENSE BLOCK ########################
2 # The Original Code is mozilla.org code.
3 #
4 # The Initial Developer of the Original Code is
5 # Netscape Communications Corporation.
6 # Portions created by the Initial Developer are Copyright (C) 1998
7 # the Initial Developer. All Rights Reserved.
8 #
9 # Contributor(s):
10 # Mark Pilgrim - port to Python
11 #
12 # This library is free software; you can redistribute it and/or
13 # modify it under the terms of the GNU Lesser General Public
14 # License as published by the Free Software Foundation; either
15 # version 2.1 of the License, or (at your option) any later version.
16 #
17 # This library is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 # Lesser General Public License for more details.
21 #
22 # You should have received a copy of the GNU Lesser General Public
23 # License along with this library; if not, write to the Free Software
24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
25 # 02110-1301 USA
26 ######################### END LICENSE BLOCK #########################
27
28 import sys
29 from .mbcharsetprober import MultiByteCharSetProber
30 from .codingstatemachine import CodingStateMachine
31 from .chardistribution import SJISDistributionAnalysis
32 from .jpcntx import SJISContextAnalysis
33 from .mbcssm import SJISSMModel
34 from . import constants
35
36
37 class SJISProber(MultiByteCharSetProber):
38 def __init__(self):
39 MultiByteCharSetProber.__init__(self)
40 self._mCodingSM = CodingStateMachine(SJISSMModel)
41 self._mDistributionAnalyzer = SJISDistributionAnalysis()
42 self._mContextAnalyzer = SJISContextAnalysis()
43 self.reset()
44
45 def reset(self):
46 MultiByteCharSetProber.reset(self)
47 self._mContextAnalyzer.reset()
48
49 def get_charset_name(self):
50 return self._mContextAnalyzer.get_charset_name()
51
52 def feed(self, aBuf):
53 aLen = len(aBuf)
54 for i in range(0, aLen):
55 codingState = self._mCodingSM.next_state(aBuf[i])
56 if codingState == constants.eError:
57 if constants._debug:
58 sys.stderr.write(self.get_charset_name()
59 + ' prober hit error at byte ' + str(i)
60 + '\n')
61 self._mState = constants.eNotMe
62 break
63 elif codingState == constants.eItsMe:
64 self._mState = constants.eFoundIt
65 break
66 elif codingState == constants.eStart:
67 charLen = self._mCodingSM.get_current_charlen()
68 if i == 0:
69 self._mLastChar[1] = aBuf[0]
70 self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:],
71 charLen)
72 self._mDistributionAnalyzer.feed(self._mLastChar, charLen)
73 else:
74 self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3
75 - charLen], charLen)
76 self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1],
77 charLen)
78
79 self._mLastChar[0] = aBuf[aLen - 1]
80
81 if self.get_state() == constants.eDetecting:
82 if (self._mContextAnalyzer.got_enough_data() and
83 (self.get_confidence() > constants.SHORTCUT_THRESHOLD)):
84 self._mState = constants.eFoundIt
85
86 return self.get_state()
87
88 def get_confidence(self):
89 contxtCf = self._mContextAnalyzer.get_confidence()
90 distribCf = self._mDistributionAnalyzer.get_confidence()
91 return max(contxtCf, distribCf)