| | 29 | |
| | 30 | @ptext = t |
| | 31 | @nponly = false |
| | 32 | @toks = [] |
| | 33 | end |
| | 34 | |
| | 35 | def nponly? |
| | 36 | return @nponly |
| | 37 | end |
| | 38 | |
| | 39 | def consumeAll |
| | 40 | @nponly = true |
| | 41 | loop do |
| | 42 | break if (!consume) |
| | 43 | end |
| | 44 | end |
| | 45 | |
| | 46 | def consume |
| | 47 | @ptext.gsub!(/^[ぁ-んァ-ヴー\-~☆!?!?・'()a-zA-Z&\s;./]+/, '') |
| | 48 | if $& |
| | 49 | @toks << SyoboToken.new($&, ST_OTHER) |
| | 50 | STDERR.puts " NP \"#{$&}\"".tosjis |
| | 51 | return true |
| | 52 | end |
| | 53 | |
| | 54 | @ptext.gsub!(/^[一-龠々〆]{2,}/, '') |
| | 55 | if $& |
| | 56 | @toks << SyoboToken.new($&, ST_KANJI) |
| | 57 | @nponly = false |
| | 58 | STDERR.puts " 漢 \"#{$&}\"".tosjis |
| | 59 | return true |
| | 60 | end |
| | 61 | |
| | 62 | @ptext.gsub!(/^[一-龠々〆][ぁ-ん]+/, '') |
| | 63 | if $& |
| | 64 | @toks << SyoboToken.new($&, ST_KANJIKANA) |
| | 65 | @nponly = false |
| | 66 | STDERR.puts " 和1 \"#{$&}\"".tosjis |
| | 67 | return true |
| | 68 | end |
| | 69 | |
| | 70 | @ptext.gsub!(/^[一-龠々〆][ぁ-んァ-ヴー]+/, '') |
| | 71 | if $& |
| | 72 | @toks << SyoboToken.new($&, ST_KANJIKANA) |
| | 73 | @nponly = false |
| | 74 | STDERR.puts " 和2 \"#{$&}\"".tosjis |
| | 75 | return true |
| | 76 | end |
| | 77 | |
| | 78 | @ptext.gsub!(/^([一-龠々〆])$/, '') |
| | 79 | if $& |
| | 80 | @toks << SyoboToken.new($1, ST_KANJI) |
| | 81 | @nponly = false |
| | 82 | STDERR.puts " 単 \"#{$&}\"".tosjis |
| | 83 | return true |
| | 84 | end |
| | 85 | |
| | 86 | @ptext.gsub!(/^[0-9]+/, '') |
| | 87 | if $& |
| | 88 | @toks << SyoboToken.new($&, ST_OTHER) |
| | 89 | @nponly = false |
| | 90 | STDERR.puts " 数 \"#{$&}\"".tosjis |
| | 91 | return true |
| | 92 | end |
| | 93 | |
| | 94 | return false if @ptext.length < 1 |
| | 95 | |
| | 96 | puts " ** WARNING **" |
| | 97 | |
| | 98 | return false |