blib/lib/Lingua/ZH/Summarize.pm

filter: Statementstmt 83.3% 10 / 12 Branchbran n/a Conditioncond n/a MC/DCmcdc n/a Subroutinesub 100% 4 / 4 Podpod n/a totaltotal 87.5% 14 / 16 CC 1 SCAR 0.7
CC
1
Cov
83%
BEGIN
1.3
BEGIN
1.0
BEGIN
1.0
CRAP 1.1
↑ Summary
1 # $File: //member/autrijus/Lingua-ZH-Summarize/Summarize.pm $ $Author: autrijus $
2 # $Revision: #2 $ $Change: 3687 $ $DateTime: 2003/01/20 07:22:40 $
3  
4 package Lingua::ZH::Summarize;
5 $Lingua::ZH::Summarize::VERSION = '0.01';
6  
7 1 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 1 ▶use vars qw($VERSION @ISA @EXPORT);
Subroutine BEGIN: called CC=1 SCAR=0.0
9  
10 1 ▶use Exporter;
Subroutine BEGIN: called CC=1 SCAR=0.0
11 0 ▶use Lingua::ZH::Toke;
Subroutine BEGIN: called CC=1 SCAR=2.6
12  
13 =head1 NAME
14  
15 Lingua::ZH::Summarize - Summarizing bodies of Chinese text
16  
17 =head1 SYNOPSIS
18  
19 use Lingua::ZH::Summarize;
20  
21 print summarize( $text ); # Easy, no? :-)
22 print summarize( $text, maxlength => 500 ); # 500-byte summary
23 print summarize( $text, wrap => 75 ); # Wrap output to 75 col.
24  
25 =head1 DESCRIPTION
26  
27 This is a simple module which makes an unscientific effort at
28 summarizing Chinese text. It recognizes simple patterns which look like
29 statements, abridges them, and concatenates them into something vaguely
30 resembling a summary. It needs more work on large bodies of text, but
31 it seems to have a decent effect on small inputs at the moment.
32  
33 Lingua::ZH::Summarize exports one function, C<summarize()>, which takes
34 the text to summarize as its first argument, and any number of optional
35 directives in C<name =E<gt> value> form. The options it'll take are:
36  
37 =over
38  
39 =item maxlength
40  
41 Specifies the maximum length, in bytes, of the generated summary.
42  
43 =item wrap
44  
45 Prettyprints the summary output by wrapping it to the number of columns
46 which you specify. This requires the L<Lingua::ZH::Wrap> module.
47  
48 =back
49  
50 Needless to say, this is a very simple and not terribly universally
51 effective scheme, but it's good enough for a first draft, and I'll bang
52 on it more later. Like I said, it's not a scientific approach to the
53 problem, but it's better than nothing.
54  
55 =cut
56  
57 @ISA = qw(Exporter);
58 @EXPORT = qw(summarize);
59  
60 my %punct = map { $_ => $_ } qw(¡C ¡H ¡I ¡F ...);
61 $punct{$_} = '¡C' for qw(¡A ¡Ð);
62  
63 my %key = map { $_ => 1 } qw(¬O ·| ´¿ ±N ¯à ¦³);
64 my %stop = (
65 %key, map { $_ => 1 } qw(
66 ªº ¸Ì ©M »P ¤Î ¦~ ¤ë ¤é ®É ¤À ¬í ¥i ¹ï ©ó ¦ý ¤] ¥B ©Î ¤¤ ¦Ó ¬° ¥s
67 )
68 );
69  
70 my $is_utf8;
71 sub import {
72 my $class = shift;
73  
74 if ($_[0] eq 'utf8' and !$is_utf8++) {
75 Lingua::ZH::Toke->import(@_);
76  
77 require Encode;
78 %key = map { Encode::decode( big5 => $_ ) => 1 } keys %key;
79 %stop = map { Encode::decode( big5 => $_ ) => 1 } keys %stop;
80 %punct = map {
81 Encode::decode( big5 => $_ ) =>
82 Encode::decode( big5 => $punct{$_} )
83 } keys %punct;
84 }
85 }
86  
87 sub summarize {
88 my ($text, %options) = @_;
89  
90 # Run each filter over the text.
91 return unless $text;
92  
93 # Strip whitespace and formatting out of the text.
94 $text =~ s/^\s+//;
95 $text =~ s/\s+/ /sg;
96 $text =~ s/\s+$//;
97  
98 unless (exists $options{maxlength} and $options{maxlength} > 0) {
99 $options{maxlength} = log(length($text)) * 30;
100 }
101  
102 ### Here's where the interesting logic happens.
103  
104 my $sentence = Lingua::ZH::Toke->new($text);
105  
106 # First we break it into sentence pieces. Kind of. Sort of.
107  
108 my $summary = '';
109 my $flag = 1;
110  
111 <$sentence> unless $sentence->[0][0];
112  
113 while (my ($chunk, $punct) = (scalar <$sentence>, scalar <$sentence>)) {
114 ($flag = $punct{$punct}, next) unless $flag;
115 $flag = $punct{$punct} or next;
116  
117 next unless length($chunk) > 10;
118 next unless $chunk =~ /.+(?:\Q¬O\E|\Q·|\E|\Q´¿\E|\Q±N\E|\Q¯à\E|\Q¦³\E)/;
119 next if $stop{substr($chunk, 0, 2)} or $stop{substr($chunk, -2)};
120  
121 $summary .= $chunk . $punct{$punct};
122  
123 last if length($summary) >= $options{maxlength};
124 }
125  
126 ### Done! Do any necessary postprocessing before returning.
127  
128 return $summary unless $options{wrap};
129  
130 # Prettyprint the summary to make it look nice on a terminal, if requested.
131  
132 require Lingua::ZH::Wrap;
133  
134 $summary = Encode::encode(big5 => $summary) if $is_utf8;
135 $summary = Lingua::ZH::Wrap::wrap(
136 $summary, $options{wrap} || 72, 1
137 );
138 $summary = Encode::decode(big5 => $summary) if $is_utf8;
139  
140 return $summary;
141 }
142  
143 1;
144  
145 =head1 SEE ALSO
146  
147 L<Lingua::ZH::Toke>, L<Lingua::ZH::Wrap>, L<Lingua::EN::Summarize>
148  
149 =head1 ACKNOWLEDGEMENTS
150  
151 Algorithm adapted from the L<Lingua::EN::Summarize> module by
152 Dennis Taylor, E<lt>dennis@funkplanet.comE<gt>.
153  
154 =head1 AUTHORS
155  
156 Autrijus Tang E<lt>autrijus@autrijus.orgE<gt>
157  
158 =head1 COPYRIGHT
159  
160 Copyright 2003 by Autrijus Tang E<lt>autrijus@autrijus.orgE<gt>.
161  
162 This program is free software; you can redistribute it and/or modify it
163 under the same terms as Perl itself.
164  
165 See L<http://www.perl.com/perl/misc/Artistic.html>
166  
167 =cut
↑ Summary