| 1 | #-*-perl-*- | ||
| 2 | # | ||
| 3 | # Copyright (C) 2004 Jörg Tiedemann <joerg@stp.ling.uu.se> | ||
| 4 | # | ||
| 5 | # This program is free software; you can redistribute it and/or modify | ||
| 6 | # it under the terms of the GNU General Public License as published by | ||
| 7 | # the Free Software Foundation; either version 2 of the License, or | ||
| 8 | # (at your option) any later version. | ||
| 9 | # | ||
| 10 | # This program is distributed in the hope that it will be useful, | ||
| 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 13 | # GNU General Public License for more details. | ||
| 14 | # | ||
| 15 | # You should have received a copy of the GNU General Public License | ||
| 16 | # along with this program; if not, write to the Free Software | ||
| 17 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
| 18 | # | ||
| 19 | #---------------------------------------------------------------------------- | ||
| 20 | # | ||
| 21 | # | ||
| 22 | |||
| 23 | |||
| 24 | package Uplug::CoocStat; | ||
| 25 | |||
| 26 | use strict; | ||
| 27 | |||
| 28 | use vars qw(@ISA); | ||
| 29 | @ISA; | ||
| 30 | |||
| 31 | sub new{ | ||
| 32 | |||
| 33 | my $class=shift; | ||
| 34 | my ($m)=@_; | ||
| 35 | |||
| 36 | if ($m=~/dice/i){return Uplug::CoocStat::Dice->new();} | ||
| 37 | if ($m=~/jaccard/i){return Uplug::CoocStat::Jaccard->new();} | ||
| 38 | if ($m=~/t-?score/i){return Uplug::CoocStat::tscore->new();} | ||
| 39 | if ($m=~/(ll|log\s*like)/i){return Uplug::CoocStat::LL->new();} | ||
| 40 | if ($m=~/(pmi|pointwise)/i){return Uplug::CoocStat::PMI->new();} | ||
| 41 | if ($m=~/(tmi|mutual)/i){return Uplug::CoocStat::I->new();} | ||
| 42 | if ($m=~/(association|mi)/i){return Uplug::CoocStat::MI->new();} | ||
| 43 | if ($m=~/(cubic|mi3)/i){return Uplug::CoocStat::MI3->new();} | ||
| 44 | if ($m=~/phi/i){return Uplug::CoocStat::Phi->new();} | ||
| 45 | if ($m=~/chi/i){return Uplug::CoocStat::Chi2->new();} | ||
| 46 | if ($m=~/kulczinsky/i){return Uplug::CoocStat::Kulczinsky->new();} | ||
| 47 | if ($m=~/ochiai/i){return Uplug::CoocStat::Ochiai->new();} | ||
| 48 | if ($m=~/yule/i){return Uplug::CoocStat::Yule->new();} | ||
| 49 | if ($m=~/connoughy/i){return Uplug::CoocStat::McConnoughy->new();} | ||
| 50 | if ($m=~/(simple|matching)/i){return Uplug::CoocStat::Simple->new();} | ||
| 51 | return undef; | ||
| 52 | } | ||
| 53 | |||
| 54 | |||
| 55 | #------------------------------------------------------------------ | ||
| 56 | # | ||
| 57 | # frequencies: | ||
| 58 | # | word 1 not word1 | | ||
| 59 | #-------------------------------------------- | ||
| 60 | # word 2 | n11 n10 | n1x | ||
| 61 | # not word 2 | n01 n00 | n0x | ||
| 62 | #-------------------------------------------- | ||
| 63 | # | nx1 nx0 | nxx | ||
| 64 | # | ||
| 65 | # | ||
| 66 | |||
| 67 | package Uplug::CoocStat::Measure; | ||
| 68 | |||
| 69 | sub new{ | ||
| 70 | my $class=shift; | ||
| 71 | my ($measure)=@_; | ||
| 72 | my $s={}; | ||
| 73 | return bless $s,$class; | ||
| 74 | } | ||
| 75 | |||
| 76 | sub setValues{ | ||
| 77 | my $s=shift; | ||
| 78 | my ($joint,$left,$right,$total)=@_; | ||
| 79 | ($s->{n11},$s->{n1x},$s->{nx1},$s->{nxx})=@_; | ||
| 80 | } | ||
| 81 | |||
| 82 | sub checkValues{ | ||
| 83 | my $s=shift; | ||
| 84 | if (not $s->{n1x}){ | ||
| 85 | $s->{ERROR}='Left marginal should not be zero!'; | ||
| 86 | return 0; | ||
| 87 | } | ||
| 88 | if (not $s->{nx1}){ | ||
| 89 | $s->{ERROR}='Right marginal should not be zero!'; | ||
| 90 | return 0; | ||
| 91 | } | ||
| 92 | if (not $s->{nxx}){ | ||
| 93 | $s->{ERROR}='Total frequency should not be zero!'; | ||
| 94 | return 0; | ||
| 95 | } | ||
| 96 | if ($s->{n11}>$s->{n1x}){ | ||
| 97 | $s->{ERROR}='Joint frequency should not be greater than left marginal!'; | ||
| 98 | return 0; | ||
| 99 | } | ||
| 100 | if ($s->{n11}>$s->{nx1}){ | ||
| 101 | $s->{ERROR}='Joint frequency should not be greater than right marginal!'; | ||
| 102 | return 0; | ||
| 103 | } | ||
| 104 | if ($s->{n1x}>$s->{nxx}){ | ||
| 105 | $s->{ERROR}='Left marginal should not be greater than the total frequency!'; | ||
| 106 | return 0; | ||
| 107 | } | ||
| 108 | if ($s->{nx1}>$s->{nxx}){ | ||
| 109 | $s->{ERROR}='Right marginal should not be greater than the total frequency!'; | ||
| 110 | return 0; | ||
| 111 | } | ||
| 112 | return 1; | ||
| 113 | } | ||
| 114 | |||
| 115 | sub compute{ | ||
| 116 | my $s=shift; | ||
| 117 | $s->setValues(@_); | ||
| 118 | if (not $s->checkValues()){ | ||
| 119 | print STDERR "# Uplug::CoocStat: ",$s->{ERROR},"\n"; | ||
| 120 | return undef; | ||
| 121 | } | ||
| 122 | return 1; | ||
| 123 | } | ||
| 124 | |||
| 125 | |||
| 126 | |||
| 127 | |||
| 128 | |||
| 129 | |||
| 130 | |||
| 131 | |||
| 132 | |||
| 133 | |||
| 134 | |||
| 135 | |||
| 136 | |||
| 137 | #----------------------------------------- | ||
| 138 | # Dice Coefficient | ||
| 139 | # | ||
| 140 | # Dice = 2*P(x,y) / (P(x)+P(y)) | ||
| 141 | # = 2*joint / (left+right) | ||
| 142 | |||
| 143 | |||
| 144 | package Uplug::CoocStat::Dice; | ||
| 145 | |||
| 146 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Measure ); | ||
| 147 | |||
| 148 | sub compute{ | ||
| 149 | my $s=shift; | ||
| 150 | if ($s->SUPER::compute(@_)){ | ||
| 151 | return 2*$s->{n11}/($s->{n1x}+$s->{nx1}); | ||
| 152 | } | ||
| 153 | return undef; | ||
| 154 | } | ||
| 155 | |||
| 156 | #----------------------------------------- | ||
| 157 | # Jaccard Coefficient | ||
| 158 | # | ||
| 159 | |||
| 160 | |||
| 161 | package Uplug::CoocStat::Jaccard; | ||
| 162 | |||
| 163 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Measure ); | ||
| 164 | |||
| 165 | sub compute{ | ||
| 166 | my $s=shift; | ||
| 167 | if ($s->SUPER::compute(@_)){ | ||
| 168 | return $s->{n11}/($s->{n1x}+$s->{nx1}-$s->{n11}); | ||
| 169 | } | ||
| 170 | return undef; | ||
| 171 | } | ||
| 172 | |||
| 173 | |||
| 174 | #----------------------------------------- | ||
| 175 | # point-wise mututal information (PMI) | ||
| 176 | # | ||
| 177 | # PMI = log( P(x,y) / (P(x)p(y)) ) | ||
| 178 | |||
| 179 | |||
| 180 | package Uplug::CoocStat::PMI; | ||
| 181 | |||
| 182 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Measure ); | ||
| 183 | |||
| 184 | sub compute{ | ||
| 185 | my $s=shift; | ||
| 186 | if ($s->SUPER::compute(@_)){ | ||
| 187 | my $score= | ||
| 188 | (($s->{n11}/$s->{n1x})/$s->{nx1})*$s->{nxx}; | ||
| 189 | if ($score<=0){ | ||
| 190 | print STDERR "Attempt to take log of 0 or negative value."; | ||
| 191 | return undef; | ||
| 192 | } | ||
| 193 | return log($score)/log(2); | ||
| 194 | } | ||
| 195 | return undef; | ||
| 196 | } | ||
| 197 | |||
| 198 | |||
| 199 | #----------------------------------------- | ||
| 200 | # t-score | ||
| 201 | # | ||
| 202 | # t-score = (P(x,y)-P(x)P(y)) / sqrt( 1/K * P(x,y) ) | ||
| 203 | |||
| 204 | |||
| 205 | package Uplug::CoocStat::tscore; | ||
| 206 | |||
| 207 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Measure ); | ||
| 208 | |||
| 209 | |||
| 210 | sub compute{ | ||
| 211 | my $s=shift; | ||
| 212 | if ($s->SUPER::compute(@_)){ | ||
| 213 | return sqrt($s->{n11})-($s->{n1x}*$s->{nx1})/ | ||
| 214 | (sqrt($s->{n11})*($s->{nxx})); | ||
| 215 | } | ||
| 216 | return undef; | ||
| 217 | } | ||
| 218 | |||
| 219 | sub compute_old{ | ||
| 220 | my $s=shift; | ||
| 221 | if ($s->SUPER::compute(@_)){ | ||
| 222 | |||
| 223 | my $Pxy=$s->{n11}/$s->{nxx}; | ||
| 224 | my $Px=$s->{nx1}/$s->{nxx}; | ||
| 225 | my $Py=$s->{n1x}/$s->{nxx}; | ||
| 226 | |||
| 227 | return ($Pxy-($Px*$Py))/sqrt($Pxy/$s->{nxx}); | ||
| 228 | } | ||
| 229 | return undef; | ||
| 230 | } | ||
| 231 | |||
| 232 | |||
| 233 | #----------------------------------------- | ||
| 234 | # the PHI coefficient | ||
| 235 | # | ||
| 236 | |||
| 237 | package Uplug::CoocStat::Phi; | ||
| 238 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Measure ); | ||
| 239 | |||
| 240 | |||
| 241 | sub setValues{ | ||
| 242 | my $s=shift; | ||
| 243 | $s->SUPER::setValues(@_); | ||
| 244 | |||
| 245 | $s->{n11}=$s->{n11}; | ||
| 246 | $s->{n10}=$s->{n1x}-$s->{n11}; | ||
| 247 | $s->{n01}=$s->{nx1}-$s->{n11}; | ||
| 248 | $s->{n00}=$s->{nxx}-$s->{n11}-$s->{n10}-$s->{n01}; | ||
| 249 | $s->{n0x}=$s->{nxx}-$s->{n1x}; | ||
| 250 | $s->{nx0}=$s->{nxx}-$s->{nx1}; | ||
| 251 | } | ||
| 252 | |||
| 253 | sub compute{ | ||
| 254 | my $s=shift; | ||
| 255 | if ($s->SUPER::compute(@_)){ | ||
| 256 | return ((($s->{n11}*$s->{n00})-($s->{n10}*$s->{n01}))**2)/ | ||
| 257 | ($s->{n1x}*$s->{nx1}*$s->{n0x}*$s->{nx0}); | ||
| 258 | } | ||
| 259 | return undef; | ||
| 260 | } | ||
| 261 | |||
| 262 | |||
| 263 | |||
| 264 | #----------------------------------------- | ||
| 265 | # the log-likelihood (ll) | ||
| 266 | # | ||
| 267 | |||
| 268 | package Uplug::CoocStat::LL; | ||
| 269 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 270 | |||
| 271 | |||
| 272 | sub checkValues{ | ||
| 273 | my $s=shift; | ||
| 274 | if (not $s->SUPER::checkValues(@_)){return 0;} | ||
| 275 | if (not $s->{n10}){ | ||
| 276 | $s->{ERROR}='n10 should not be zero!'; | ||
| 277 | return 0; | ||
| 278 | } | ||
| 279 | if (not $s->{n01}){ | ||
| 280 | $s->{ERROR}='n01 should not be zero!'; | ||
| 281 | return 0; | ||
| 282 | } | ||
| 283 | if (not $s->{n00}){ | ||
| 284 | $s->{ERROR}='n00 should not be zero!'; | ||
| 285 | return 0; | ||
| 286 | } | ||
| 287 | return 1; | ||
| 288 | } | ||
| 289 | |||
| 290 | |||
| 291 | sub compute{ | ||
| 292 | my $s=shift; | ||
| 293 | if ($s->SUPER::compute(@_)){ | ||
| 294 | my $ll= | ||
| 295 | $s->{n11}*log($s->{n11})+ | ||
| 296 | $s->{n10}*log($s->{n10})+ | ||
| 297 | $s->{n01}*log($s->{n01})+ | ||
| 298 | $s->{n00}*log($s->{n00}) | ||
| 299 | -($s->{n1x}*log($s->{n1x})) | ||
| 300 | -($s->{nx1}*log($s->{nx1})) | ||
| 301 | -($s->{nx0}*log($s->{nx0})) | ||
| 302 | -($s->{n0x}*log($s->{n0x})) | ||
| 303 | +($s->{nxx}*log($s->{n11}+$s->{nxx})); | ||
| 304 | return $ll; | ||
| 305 | |||
| 306 | # my $ll= | ||
| 307 | # $s->{n11}*log($s->{n11})+ | ||
| 308 | # $s->{n10}*log($s->{n10})+ | ||
| 309 | # $s->{n01}*log($s->{n01})+ | ||
| 310 | # $s->{n00}*log($s->{n00}) | ||
| 311 | # -($s->{n11}+$s->{n10})*log($s->{n11}+$s->{n10}) | ||
| 312 | # -($s->{n11}+$s->{n01})*log($s->{n11}+$s->{n01}) | ||
| 313 | # -($s->{n10}+$s->{n00})*log($s->{n10}+$s->{n00}) | ||
| 314 | # -($s->{n01}+$s->{n00})*log($s->{n01}+$s->{n00}) | ||
| 315 | # +($s->{n11}+$s->{n10}+$s->{n01}+$s->{n00})* | ||
| 316 | # log($s->{n11}+$s->{n10}+$s->{n01}+$s->{n00}); | ||
| 317 | # return $ll; | ||
| 318 | } | ||
| 319 | return undef; | ||
| 320 | } | ||
| 321 | |||
| 322 | |||
| 323 | #----------------------------------------- | ||
| 324 | # Cubic Association Ratio (MI3) | ||
| 325 | # | ||
| 326 | |||
| 327 | package Uplug::CoocStat::MI3; | ||
| 328 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 329 | |||
| 330 | sub compute{ | ||
| 331 | my $s=shift; | ||
| 332 | if ($s->SUPER::compute(@_)){ | ||
| 333 | return log(($s->{n11}**3)/ | ||
| 334 | (($s->{n11}+$s->{n10})* | ||
| 335 | ($s->{n11}+$s->{n01})))/log(2); | ||
| 336 | } | ||
| 337 | } | ||
| 338 | |||
| 339 | |||
| 340 | #----------------------------------------- | ||
| 341 | # Association Ratio (MI) | ||
| 342 | # | ||
| 343 | |||
| 344 | package Uplug::CoocStat::MI; | ||
| 345 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 346 | |||
| 347 | sub compute{ | ||
| 348 | my $s=shift; | ||
| 349 | if ($s->SUPER::compute(@_)){ | ||
| 350 | return log(($s->{n11})/ | ||
| 351 | (($s->{n11}+$s->{n10})* | ||
| 352 | ($s->{n11}+$s->{n01})))/log(2); | ||
| 353 | } | ||
| 354 | } | ||
| 355 | |||
| 356 | |||
| 357 | #----------------------------------------- | ||
| 358 | # Kulczinsky Coefficient | ||
| 359 | # | ||
| 360 | |||
| 361 | package Uplug::CoocStat::Kulczinsky; | ||
| 362 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 363 | |||
| 364 | sub compute{ | ||
| 365 | my $s=shift; | ||
| 366 | if ($s->SUPER::compute(@_)){ | ||
| 367 | return ($s->{n11}/2)* | ||
| 368 | (1/($s->{n11}+$s->{n10})+1/($s->{n11}+$s->{n01})); | ||
| 369 | } | ||
| 370 | } | ||
| 371 | |||
| 372 | |||
| 373 | |||
| 374 | #----------------------------------------- | ||
| 375 | # Ochiai Coefficient | ||
| 376 | # | ||
| 377 | |||
| 378 | package Uplug::CoocStat::Ochiai; | ||
| 379 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 380 | |||
| 381 | sub compute{ | ||
| 382 | my $s=shift; | ||
| 383 | if ($s->SUPER::compute(@_)){ | ||
| 384 | return ($s->{n11})/ | ||
| 385 | (sqrt(($s->{n11}+$s->{n10})*($s->{n11}+$s->{n01}))); | ||
| 386 | } | ||
| 387 | } | ||
| 388 | |||
| 389 | |||
| 390 | |||
| 391 | #----------------------------------------- | ||
| 392 | # Yule Coefficient | ||
| 393 | # | ||
| 394 | |||
| 395 | package Uplug::CoocStat::Yule; | ||
| 396 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 397 | |||
| 398 | sub compute{ | ||
| 399 | my $s=shift; | ||
| 400 | if ($s->SUPER::compute(@_)){ | ||
| 401 | return ($s->{n11}*$s->{n00}-$s->{n10}*$s->{n01})/ | ||
| 402 | (($s->{n11}*$s->{n00})+($s->{n10}*$s->{n01})); | ||
| 403 | } | ||
| 404 | |||
| 405 | } | ||
| 406 | |||
| 407 | |||
| 408 | #----------------------------------------- | ||
| 409 | # McConnoughy Coefficient | ||
| 410 | # | ||
| 411 | |||
| 412 | package Uplug::CoocStat::McConnoughy; | ||
| 413 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 414 | |||
| 415 | sub compute{ | ||
| 416 | my $s=shift; | ||
| 417 | if ($s->SUPER::compute(@_)){ | ||
| 418 | return ($s->{n11}**2-$s->{n10}*$s->{n01})/ | ||
| 419 | (($s->{n11}+$s->{n10})*($s->{n11}+$s->{n01})); | ||
| 420 | } | ||
| 421 | } | ||
| 422 | |||
| 423 | |||
| 424 | #----------------------------------------- | ||
| 425 | # Simple Matchink Coefficient | ||
| 426 | # | ||
| 427 | |||
| 428 | package Uplug::CoocStat::Simple; | ||
| 429 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 430 | |||
| 431 | sub compute{ | ||
| 432 | my $s=shift; | ||
| 433 | if ($s->SUPER::compute(@_)){ | ||
| 434 | return ($s->{n11}+$s->{n10})/ | ||
| 435 | ($s->{n11}+$s->{n10}+$s->{n01}+$s->{n00}); | ||
| 436 | } | ||
| 437 | } | ||
| 438 | |||
| 439 | |||
| 440 | #----------------------------------------- | ||
| 441 | # Chi2 | ||
| 442 | # | ||
| 443 | # taken from nsp (Ted Pedersen) | ||
| 444 | # | ||
| 445 | |||
| 446 | package Uplug::CoocStat::Chi2; | ||
| 447 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::Phi ); | ||
| 448 | |||
| 449 | sub compute{ | ||
| 450 | my $s=shift; | ||
| 451 | if ($s->SUPER::compute(@_)){ | ||
| 452 | |||
| 453 | my $m11 = $s->{n1x} * $s->{nx1} / $s->{nxx}; | ||
| 454 | my $m12 = $s->{n1x} * $s->{nx0} / $s->{nxx}; | ||
| 455 | my $m21 = $s->{n0x} * $s->{nx1} / $s->{nxx}; | ||
| 456 | my $m22 = $s->{n0x} * $s->{nx0} / $s->{nxx}; | ||
| 457 | |||
| 458 | my $Xsquare = 0; | ||
| 459 | if ($m11 == 0 || $m12 == 0 || $m21 == 0 || $m22 == 0){return undef;} | ||
| 460 | |||
| 461 | $Xsquare += ( ( $s->{n11} - $m11 ) ** 2 ) / $m11; | ||
| 462 | $Xsquare += ( ( $s->{n10} - $m12 ) ** 2 ) / $m12; | ||
| 463 | $Xsquare += ( ( $s->{n01} - $m21 ) ** 2 ) / $m21; | ||
| 464 | $Xsquare += ( ( $s->{n00} - $m22 ) ** 2 ) / $m22; | ||
| 465 | |||
| 466 | return $Xsquare; | ||
| 467 | } | ||
| 468 | } | ||
| 469 | |||
| 470 | |||
| 471 | #----------------------------------------- | ||
| 472 | # mututal information | ||
| 473 | # | ||
| 474 | # taken from nsp (Ted Pedersen) | ||
| 475 | # | ||
| 476 | |||
| 477 | package Uplug::CoocStat::I; | ||
| 478 | use vars qw(@ISA); @ISA = qw( Uplug::CoocStat::LL ); | ||
| 479 | |||
| 480 | sub compute{ | ||
| 481 | my $s=shift; | ||
| 482 | if ($s->SUPER::compute(@_)){ | ||
| 483 | |||
| 484 | my $m11 = $s->{n1x} * $s->{nx1} / $s->{nxx}; | ||
| 485 | my $m12 = $s->{n1x} * $s->{nx0} / $s->{nxx}; | ||
| 486 | my $m21 = $s->{n0x} * $s->{nx1} / $s->{nxx}; | ||
| 487 | my $m22 = $s->{n0x} * $s->{nx0} / $s->{nxx}; | ||
| 488 | |||
| 489 | my $tmi=0; | ||
| 490 | $tmi += $s->{n11}/$s->{nxx} * log ( $s->{n11} / $m11 ) / log 2; | ||
| 491 | $tmi += $s->{n10}/$s->{nxx} * log ( $s->{n10} / $m12 ) / log 2; | ||
| 492 | $tmi += $s->{n01}/$s->{nxx} * log ( $s->{n01} / $m21 ) / log 2; | ||
| 493 | $tmi += $s->{n00}/$s->{nxx} * log ( $s->{n00} / $m22 ) / log 2; | ||
| 494 | return $tmi; | ||
| 495 | } | ||
| 496 | return undef; | ||
| 497 | } | ||
| 498 | |||
| 499 | |||
| 500 | 1; |