| 1 | ## -*- Mode: CPerl -*- | ||
| 2 | ## File: DTA::CAB::Analyzer::DTAClean.pm | ||
| 3 | ## Author: Bryan Jurish <moocow@cpan.org> | ||
| 4 | ## Description: Chain::DTA cleanup (prune sensitive and redundant data from document) | ||
| 5 | |||
| 6 | package DTA::CAB::Analyzer::DTAClean; | ||
| 7 | use DTA::CAB::Analyzer; | ||
| 8 | use Carp; | ||
| 9 | use strict; | ||
| 10 | |||
| 11 | ##============================================================================== | ||
| 12 | ## Constants | ||
| 13 | ##============================================================================== | ||
| 14 | our @ISA = qw(DTA::CAB::Analyzer); | ||
| 15 | |||
| 16 | ##============================================================================== | ||
| 17 | ## Constructors etc. | ||
| 18 | ##============================================================================== | ||
| 19 | |||
| 20 | ## $obj = CLASS_OR_OBJ->new(%args) | ||
| 21 | ## + object structure: HASH | ||
| 22 | sub new { | ||
| 23 | my $that = shift; | ||
| 24 | return $that->SUPER::new( | ||
| 25 | ##-- security | ||
| 26 | label => 'clean', | ||
| 27 | forceClean => 1, ##-- always run analyzeClean() regardless of options; also checked in analyzeClean() itself | ||
| 28 | cleanPublic => 1, ##-- cleanup level: 0=verbose, 1=terse (for public wrapper "clean" parameter) | ||
| 29 | |||
| 30 | ##-- user args | ||
| 31 | @_, | ||
| 32 | ); | ||
| 33 | } | ||
| 34 | |||
| 35 | ##============================================================================== | ||
| 36 | ## Methods: I/O | ||
| 37 | ##============================================================================== | ||
| 38 | |||
| 39 | ##============================================================================== | ||
| 40 | ## Methods: Persistence | ||
| 41 | ##============================================================================== | ||
| 42 | |||
| 43 | ##============================================================================== | ||
| 44 | ## Methods: Analysis | ||
| 45 | ##============================================================================== | ||
| 46 | |||
| 47 | ##------------------------------------------------------------------------ | ||
| 48 | ## Methods: Analysis: v1.x: Utils | ||
| 49 | |||
| 50 | ## $bool = $anl->doAnalyze(\%opts, $name) | ||
| 51 | ## + alias for $anl->can("analyze${name}") && (!exists($opts{"doAnalyze${name}"}) || $opts{"doAnalyze${name}"}) | ||
| 52 | ## + override checks $anl->{forceClean} flag | ||
| 53 | sub doAnalyze { | ||
| 54 | my ($anl,$opts,$name) = @_; | ||
| 55 | return 1 if ($anl->{forceClean} && $name eq 'Clean'); ##-- always clean if requested | ||
| 56 | return $anl->SUPER::doAnalyze($opts,$name); | ||
| 57 | } | ||
| 58 | |||
| 59 | |||
| 60 | ##------------------------------------------------------------------------ | ||
| 61 | ## Methods: Analysis: v1.x: API | ||
| 62 | |||
| 63 | ## $doc = $ach->analyzeDocument($doc,\%opts) | ||
| 64 | ## + analyze a DTA::CAB::Document $doc | ||
| 65 | ## + top-level API routine | ||
| 66 | ## + INHERITED from DTA::CAB::Analyzer | ||
| 67 | |||
| 68 | ## $doc = $ach->analyzeTypes($doc,$types,\%opts) | ||
| 69 | ## + perform type-wise analysis of all (text) types in $doc->{types} | ||
| 70 | ## + Chain default calls $a->analyzeTypes for each analyzer $a in the chain | ||
| 71 | ## + INHERITED from DTA::CAB::Chain | ||
| 72 | |||
| 73 | ## $doc = $ach->analyzeTokens($doc,\%opts) | ||
| 74 | ## + perform token-wise analysis of all tokens $doc->{body}[$si]{tokens}[$wi] | ||
| 75 | ## + default implementation just shallow copies tokens in $doc->{types} | ||
| 76 | ## + INHERITED from DTA::CAB::Analyzer | ||
| 77 | |||
| 78 | ## $doc = $ach->analyzeSentences($doc,\%opts) | ||
| 79 | ## + perform sentence-wise analysis of all sentences $doc->{body}[$si] | ||
| 80 | ## + Chain default calls $a->analyzeSentences for each analyzer $a in the chain | ||
| 81 | ## + INHERITED from DTA::CAB::Chain | ||
| 82 | |||
| 83 | ## $doc = $ach->analyzeLocal($doc,\%opts) | ||
| 84 | ## + perform local document-level analysis of $doc | ||
| 85 | ## + Chain default calls $a->analyzeLocal for each analyzer $a in the chain | ||
| 86 | ## + INHERITED from DTA::CAB::Chain | ||
| 87 | |||
| 88 | ## $doc = $ach->analyzeClean($doc,\%opts) | ||
| 89 | ## + cleanup any temporary data associated with $doc | ||
| 90 | ## + override deletes black-listed keys | ||
| 91 | sub analyzeClean { | ||
| 92 | my ($ach,$doc,$opts) = @_; | ||
| 93 | |||
| 94 | ##-- prune output (black-list) | ||
| 95 | my $cleanPublic = (exists($opts->{cleanPublic}) ? $opts->{cleanPublic} : $ach->{cleanPublic}) || 0; | ||
| 96 | $ach->debug("analyzeClean(): cleanPublic=$cleanPublic"); | ||
| 97 | |||
| 98 | ##-- black-list | ||
| 99 | my @prune_keys = (qw(morph), | ||
| 100 | ($cleanPublic ? qw(tokpp toka lts rw eqphox dmoot) : qw()), | ||
| 101 | ); | ||
| 102 | my ($s,$w); | ||
| 103 | foreach $s (@{$doc->{body}}) { | ||
| 104 | foreach $w (@{$s->{tokens}}) { | ||
| 105 | ##-- map 'morph' to 'hasmorph' | ||
| 106 | $w->{hasmorph} = 1 if ($w->{morph} && @{$w->{morph}}); | ||
| 107 | |||
| 108 | ##-- delete all unsafe keys | ||
| 109 | delete @$w{@prune_keys}; | ||
| 110 | delete @{$w->{moot}}{qw(analyses details)} if ($w->{moot}); ##-- moot/analyses, moot/details: also unsafe | ||
| 111 | |||
| 112 | ##-- deep cleaning of embedded 'morph' keys (for {cleanPublic=>0} and rwsub, dmootsub) | ||
| 113 | if (!$cleanPublic) { | ||
| 114 | delete @$_{@prune_keys} foreach (grep {UNIVERSAL::isa($_,'HASH')} | ||
| 115 | map {UNIVERSAL::isa($_,'ARRAY') ? @$_ : $_} | ||
| 116 | values %$w); | ||
| 117 | } | ||
| 118 | } | ||
| 119 | } | ||
| 120 | |||
| 121 | return $doc; | ||
| 122 | } | ||
| 123 | |||
| 124 | |||
| 125 | 1; ##-- be happy | ||
| 126 | |||
| 127 | __END__ |