| 1 | ## -*- Mode: CPerl; coding: utf-8; -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Format::CorpusExplorerPlugin.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Datum parser/formatter: CorpusExplorer normalization plugin | ||
| 6 | |||
| 7 | ## Format description (from Jan Oliver Rüdiger, 2019-03-19) | ||
| 8 | ## | ||
| 9 | ## > Wie würde der CorpusExplorer Texte einsenden: | ||
| 10 | ## > + Die Sätze werden mittels \n getrennt. | ||
| 11 | ## > + Die Token werden mittels \t getrennt. | ||
| 12 | ## > + Der Aufruf des Webservice geschieht nacheinander. | ||
| 13 | ## > + Der CorpusExplorer stellt sicher, dass KEINE Dokumente >1MB verschickt werden (Ablehnung vor Request). | ||
| 14 | ## > + Der CorpusExplorer stellt sicher, dass die Dokumente nacheinander und nicht parallel verschickt werden. | ||
| 15 | ## > Bei Bedarf kann auch nach nach jedem Dokument (oder) nach einer festen Anzahl (z. B. 100) eine Pause eingelegt werden. | ||
| 16 | ## > + Der CorpusExplorer stellt sicher, dass nicht mehr als XXXXX Dokumente annotiert werden. Bitte XXXXX spezifizieren. | ||
| 17 | ## > | ||
| 18 | ## > Was würde der CorpusExplorer erwarten: | ||
| 19 | ## > + Die Sätze sind mittels \n getrennt | ||
| 20 | ## > + Die Token sind mittels \t getrennt | ||
| 21 | ## > + Es werden nur Korrekturvorschläge übermittelt. Token, die keine Korrektur benötigen | ||
| 22 | ## > (bzw. eine Korrektur nicht möglich ist), bleiben leer. z. B. \t\tKorrektur\t\t\tBeispiel\n | ||
| 23 | |||
| 24 | package DTA::CAB::Format::CorpusExplorerPlugin; | ||
| 25 | use DTA::CAB::Format; | ||
| 26 | use DTA::CAB::Datum ':all'; | ||
| 27 | use IO::File; | ||
| 28 | use Carp; | ||
| 29 | use strict; | ||
| 30 | |||
| 31 | ##============================================================================== | ||
| 32 | ## Globals | ||
| 33 | ##============================================================================== | ||
| 34 | |||
| 35 | our @ISA = qw(DTA::CAB::Format); | ||
| 36 | |||
| 37 | BEGIN { | ||
| 38 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>$_, filenameRegex=>qr/\.(?i:ceplug(?:in)?)$/) | ||
| 39 | foreach (qw(ceplugin ceplug)); | ||
| 40 | } | ||
| 41 | |||
| 42 | ##============================================================================== | ||
| 43 | ## Constructors etc. | ||
| 44 | ##============================================================================== | ||
| 45 | |||
| 46 | ## $fmt = CLASS_OR_OBJ->new(%args) | ||
| 47 | ## + object structure: assumed HASH | ||
| 48 | ## ( | ||
| 49 | ## ##---- Input | ||
| 50 | ## doc => $doc, ##-- buffered input document | ||
| 51 | ## | ||
| 52 | ## ##---- Output | ||
| 53 | ## level => $formatLevel, ##-- output formatting level: | ||
| 54 | ## ## 0: norm (terse; empty for identity-normalizations) | ||
| 55 | ## ## 1: norm (verbose) | ||
| 56 | ## #outbuf => $stringBuffer, ##-- buffered output | ||
| 57 | ## | ||
| 58 | ## ##---- Common | ||
| 59 | ## utf8 => $bool, ##-- default: 1 | ||
| 60 | ## fh => $fh, ##-- IO::Handle for read/write | ||
| 61 | ## ) | ||
| 62 | sub new { | ||
| 63 | my $that = shift; | ||
| 64 | my $fmt = bless({ | ||
| 65 | ##-- input | ||
| 66 | doc => undef, | ||
| 67 | |||
| 68 | ##-- common | ||
| 69 | utf8 => 1, | ||
| 70 | |||
| 71 | ##-- user args | ||
| 72 | @_ | ||
| 73 | }, ref($that)||$that); | ||
| 74 | return $fmt; | ||
| 75 | } | ||
| 76 | |||
| 77 | ##============================================================================== | ||
| 78 | ## Methods: Persistence | ||
| 79 | ##============================================================================== | ||
| 80 | |||
| 81 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 82 | ## + returns list of keys not to be saved | ||
| 83 | ## + override | ||
| 84 | sub noSaveKeys { | ||
| 85 | return ($_[0]->SUPER::noSaveKeys, qw(doc outbuf)); | ||
| 86 | } | ||
| 87 | |||
| 88 | ##============================================================================== | ||
| 89 | ## Methods: Input | ||
| 90 | ##============================================================================== | ||
| 91 | |||
| 92 | ##-------------------------------------------------------------- | ||
| 93 | ## Methods: Input: Input selection | ||
| 94 | |||
| 95 | ## $fmt = $fmt->fromFh($filename_or_handle) | ||
| 96 | ## + override calls fromFh_str() | ||
| 97 | sub fromFh { | ||
| 98 | return $_[0]->fromFh_str(@_[1..$#_]); | ||
| 99 | } | ||
| 100 | |||
| 101 | ## $fmt = $fmt->fromString(\$string) | ||
| 102 | ## + select input from string $string | ||
| 103 | sub fromString { | ||
| 104 | my $fmt = shift; | ||
| 105 | $fmt->close(); | ||
| 106 | return $fmt->parseCeString(ref($_[0]) ? $_[0] : \$_[0]); | ||
| 107 | } | ||
| 108 | |||
| 109 | |||
| 110 | ##-------------------------------------------------------------- | ||
| 111 | ## Methods: Input: Local | ||
| 112 | |||
| 113 | ## $fmt = $fmt->parseCeString(\$string) | ||
| 114 | sub parseCeString { | ||
| 115 | my ($fmt,$src) = @_; | ||
| 116 | no warnings qw(uninitialized); | ||
| 117 | utf8::decode($$src) if ($fmt->{utf8} && !utf8::is_utf8($$src)); | ||
| 118 | |||
| 119 | my ($toks); | ||
| 120 | my $sents = | ||
| 121 | [ | ||
| 122 | map { | ||
| 123 | {tokens=>[map {{text=>$_}} split(/\t/,($_//''))]} | ||
| 124 | } split(/\n/,$$src) | ||
| 125 | ]; | ||
| 126 | $fmt->{doc} = bless({body=>$sents}, 'DTA::CAB::Document'); | ||
| 127 | |||
| 128 | return $fmt; | ||
| 129 | } | ||
| 130 | |||
| 131 | ##-------------------------------------------------------------- | ||
| 132 | ## Methods: Input: Generic API | ||
| 133 | |||
| 134 | ## $doc = $fmt->parseDocument() | ||
| 135 | sub parseDocument { return $_[0]{doc}; } | ||
| 136 | |||
| 137 | |||
| 138 | ##============================================================================== | ||
| 139 | ## Methods: Output | ||
| 140 | ##============================================================================== | ||
| 141 | |||
| 142 | ##-------------------------------------------------------------- | ||
| 143 | ## Methods: Output: Generic | ||
| 144 | |||
| 145 | |||
| 146 | ## $type = $fmt->mimeType() | ||
| 147 | ## + default returns text/plain | ||
| 148 | sub mimeType { return 'text/plain'; } | ||
| 149 | |||
| 150 | ## $ext = $fmt->defaultExtension() | ||
| 151 | ## + returns default filename extension for this format | ||
| 152 | sub defaultExtension { return '.ceplugin'; } | ||
| 153 | |||
| 154 | ## $str = $fmt->toString() | ||
| 155 | ## $str = $fmt->toString($formatLevel) | ||
| 156 | ## + flush buffered output document to byte-string | ||
| 157 | |||
| 158 | ## $fmt_or_undef = $fmt->toFile($filename_or_handle, $formatLevel) | ||
| 159 | ## + flush buffered output document to $filename_or_handle | ||
| 160 | ## + default implementation calls $fmt->toFh() | ||
| 161 | |||
| 162 | ## $fmt_or_undef = $fmt->toFh($fh,$formatLevel) | ||
| 163 | ## + flush buffered output document to filehandle $fh | ||
| 164 | ## + default implementation calls to $fmt->formatString($formatLevel) | ||
| 165 | sub toFh { | ||
| 166 | $_[0]->DTA::CAB::Format::toFh(@_[1..$#_]); | ||
| 167 | $_[0]->setLayers(); | ||
| 168 | return $_[0]; | ||
| 169 | } | ||
| 170 | |||
| 171 | ##-------------------------------------------------------------- | ||
| 172 | ## Methods: Output: API | ||
| 173 | |||
| 174 | ## $fmt = $fmt->putDocument($doc) | ||
| 175 | ## $fmt = $fmt->putDocument($doc,\$buf) | ||
| 176 | ## + override | ||
| 177 | sub putDocument { | ||
| 178 | my ($fmt,$doc,$bufr) = @_; | ||
| 179 | #$bufr = \(my $buf='') if (!defined($bufr)); | ||
| 180 | |||
| 181 | my $level = $fmt->{level} // 0; | ||
| 182 | my ($s,$w, $wnew); | ||
| 183 | foreach $s (@{$doc->{body}}) { | ||
| 184 | $fmt->{fh}->print(join("\t", | ||
| 185 | map { | ||
| 186 | $wnew = $_->{moot} && defined($_->{moot}{word}) ? $_->{moot}{word} : $_->{text}; | ||
| 187 | ($level > 0 || $wnew ne $_->{text} ? $wnew : '') | ||
| 188 | } @{$s->{tokens}}), | ||
| 189 | "\n"); | ||
| 190 | } | ||
| 191 | return $fmt; | ||
| 192 | } | ||
| 193 | |||
| 194 | ## $fmt = $fmt->putData($data) | ||
| 195 | ## + puts raw data (uses forceDocument()) | ||
| 196 | sub putData { | ||
| 197 | $_[0]->putDocument($_[0]->forceDocument($_[1])); | ||
| 198 | } | ||
| 199 | |||
| 200 | |||
| 201 | |||
| 202 | 1; ##-- be happy | ||
| 203 | |||
| 204 | __END__ |