| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## | ||
| 3 | ## File: DTA::CAB::Format::Raw::HTTP.pm | ||
| 4 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 5 | ## Description: Datum parser: raw untokenized text (using HTTP tokenizer) | ||
| 6 | |||
| 7 | package DTA::CAB::Format::Raw::HTTP; | ||
| 8 | use DTA::CAB::Format; | ||
| 9 | use DTA::CAB::Format::TT; | ||
| 10 | use DTA::CAB::Format::Raw::Base; | ||
| 11 | use DTA::CAB::Datum ':all'; | ||
| 12 | use IO::File; | ||
| 13 | use URI; | ||
| 14 | use Encode qw(encode decode); | ||
| 15 | |||
| 16 | use LWP::UserAgent; | ||
| 17 | |||
| 18 | use Carp; | ||
| 19 | use strict; | ||
| 20 | |||
| 21 | ##============================================================================== | ||
| 22 | ## Globals | ||
| 23 | ##============================================================================== | ||
| 24 | |||
| 25 | our @ISA = qw(DTA::CAB::Format::Raw::Base); | ||
| 26 | |||
| 27 | BEGIN { | ||
| 28 | DTA::CAB::Format->registerFormat(name=>__PACKAGE__, short=>'raw-http', filenameRegex=>qr/\.(?i:raw-http|txt-http)$/); | ||
| 29 | } | ||
| 30 | |||
| 31 | ##============================================================================== | ||
| 32 | ## Constructors etc. | ||
| 33 | ##============================================================================== | ||
| 34 | |||
| 35 | ## $fmt = CLASS_OR_OBJ->new(%args) | ||
| 36 | ## + object structure: assumed HASH | ||
| 37 | ## { | ||
| 38 | ## ##-- Input | ||
| 39 | ## doc => $doc, ##-- buffered input document | ||
| 40 | ## tokurl => $url, ##-- tokenizer (default='http://kaskade.dwds.de/waste/tokenize.fcgi?m=dta&O=mr,loc') | ||
| 41 | ## txtparam => $param, ##-- text query parameter (default='t') | ||
| 42 | ## timeout => $secs, ##-- user agent timeout (default=300) | ||
| 43 | ## | ||
| 44 | ## ##-- Runtime | ||
| 45 | ## ua => $ua, ##-- low-level underlying LWP::UserAgent | ||
| 46 | ## | ||
| 47 | ## ##-- Common | ||
| 48 | ## #utf8 => $bool, ##-- utf8 mode always on | ||
| 49 | sub new { | ||
| 50 | my $that = shift; | ||
| 51 | my $fmt = bless({ | ||
| 52 | ##-- common | ||
| 53 | utf8 => 1, | ||
| 54 | |||
| 55 | ##-- input | ||
| 56 | doc => undef, | ||
| 57 | tokurl => 'http://kaskade.dwds.de/waste/tokenize.fcgi?m=dta&O=mr,loc', | ||
| 58 | txtparam => 't', | ||
| 59 | timeout => 300, | ||
| 60 | |||
| 61 | ##-- runtime | ||
| 62 | ua => undef, | ||
| 63 | |||
| 64 | ##-- user args | ||
| 65 | @_ | ||
| 66 | }, ref($that)||$that); | ||
| 67 | |||
| 68 | ##-- instantiate LWP::UserAgent | ||
| 69 | if (!defined($fmt->{ua})) { | ||
| 70 | $fmt->{ua} = LWP::UserAgent->new(timeout=>$fmt->{timeout}) | ||
| 71 | or $fmt->logconfess("could not create LWP::UserAgent: $!"); | ||
| 72 | } | ||
| 73 | |||
| 74 | return $fmt; | ||
| 75 | } | ||
| 76 | |||
| 77 | ##============================================================================== | ||
| 78 | ## Methods: Persistence | ||
| 79 | ##============================================================================== | ||
| 80 | |||
| 81 | ## @keys = $class_or_obj->noSaveKeys() | ||
| 82 | ## + returns list of keys not to be saved | ||
| 83 | ## + default just returns empty list | ||
| 84 | sub noSaveKeys { | ||
| 85 | return (shift->SUPER::noSaveKeys(), qw(doc ua)); | ||
| 86 | } | ||
| 87 | |||
| 88 | ##============================================================================== | ||
| 89 | ## Methods: Input | ||
| 90 | ##============================================================================== | ||
| 91 | |||
| 92 | ##-------------------------------------------------------------- | ||
| 93 | ## Methods: Input: Input selection | ||
| 94 | |||
| 95 | ## $fmt = $fmt->close() | ||
| 96 | sub close { | ||
| 97 | delete($_[0]{doc}); | ||
| 98 | return $_[0]->SUPER::close(@_[1..$#_]); | ||
| 99 | } | ||
| 100 | |||
| 101 | ## $fmt = $fmt->fromString( $string) | ||
| 102 | ## $fmt = $fmt->fromString(\$string) | ||
| 103 | sub fromString { | ||
| 104 | my $fmt = shift; | ||
| 105 | $fmt->close(); | ||
| 106 | return $fmt->parseRawString(ref($_[0]) ? $_[0] : \$_[0]); | ||
| 107 | } | ||
| 108 | |||
| 109 | ## $fmt = $fmt->fromFh($filename_or_handle) | ||
| 110 | ## + override calls fromFh_str() | ||
| 111 | sub fromFh { | ||
| 112 | return $_[0]->fromFh_str(@_[1..$#_]); | ||
| 113 | } | ||
| 114 | |||
| 115 | ##-------------------------------------------------------------- | ||
| 116 | ## Methods: Input: local | ||
| 117 | |||
| 118 | ## $fmt = $fmt->parseRawString(\$str) | ||
| 119 | sub parseRawString { | ||
| 120 | my ($fmt,$str) = @_; | ||
| 121 | utf8::encode($$str) if (utf8::is_utf8($$str)); | ||
| 122 | |||
| 123 | ## Use multipart/form-data to avoid implicit LF->CR+LF conversion by LWP::UserAgent (HTTP::Request::Common::POST() v6.03 / debian wheezy) | ||
| 124 | #$fmt->trace("querying $fmt->{tokurl} ..."); | ||
| 125 | my $rsp = $fmt->{ua}->post($fmt->{tokurl}, { $fmt->{txtparam}=>$$str }, 'Content-Type'=>'multipart/form-data'); | ||
| 126 | if (!$rsp || !$rsp->is_success) { | ||
| 127 | $fmt->trace("parseRawString(): error from server:\n", $rsp->as_string) if ($rsp); | ||
| 128 | $fmt->logdie("parseRawString(): error from server $fmt->{tokurl}: ", ($rsp ? $rsp->status_line : '(no response)')) | ||
| 129 | if (!$rsp || !$rsp->is_success); | ||
| 130 | } | ||
| 131 | |||
| 132 | ##-- construct & buffer document | ||
| 133 | $fmt->{doc} = DTA::CAB::Format::TT->parseTokenizerString( $rsp->content_ref ); | ||
| 134 | return $fmt; | ||
| 135 | } | ||
| 136 | |||
| 137 | ##-------------------------------------------------------------- | ||
| 138 | ## Methods: Input: Generic API | ||
| 139 | |||
| 140 | ## $doc = $fmt->parseDocument() | ||
| 141 | sub parseDocument { | ||
| 142 | return $_[0]{doc}; | ||
| 143 | } | ||
| 144 | |||
| 145 | |||
| 146 | ##============================================================================== | ||
| 147 | ## Methods: Output | ||
| 148 | ## + output not supported | ||
| 149 | ##============================================================================== | ||
| 150 | |||
| 151 | ##-------------------------------------------------------------- | ||
| 152 | ## Methods: Output: Generic | ||
| 153 | |||
| 154 | ## $type = $fmt->mimeType() | ||
| 155 | ## + default returns text/plain | ||
| 156 | sub mimeType { return 'text/plain'; } | ||
| 157 | |||
| 158 | ## $ext = $fmt->defaultExtension() | ||
| 159 | ## + returns default filename extension for this format | ||
| 160 | sub defaultExtension { return '.raw'; } | ||
| 161 | |||
| 162 | |||
| 163 | 1; ##-- be happy | ||
| 164 | |||
| 165 | __END__ |