R aggregate tocken от lemme в tm пакет

Имам зареждане и почистване на корпус в R с:

myTxt <- Corpus(DirSource("."), readerControl = list(language="lat"))
corp <- tm_map(myTxt, removeWords, c(stopwords("french")))
corp <- tm_map(corp, content_transformer(tolower))
corp <- tm_map(corp, content_transformer(removeNumbers))
corp <- tm_map(corp, removePunctuation)
corp <- tm_map(corp, removeWords, stopwords("french"))
corp <- tm_map(corp, stripWhitespace); #inspect(docs[1])
tdm <- TermDocumentMatrix(corp)

И с treetagger написах функция като:

require(koRpus)

lemmatisation <- function(my.df){
  ##my.df est un objet Corpus issu de du chargement du corpus avec tm
  print(my.df)
  dictionnaire <- data.frame()
  for(i in 1 : length(my.df)){
    lemma <- treetag(corp[[i]][[1]], treetagger = "manual", format = "obj", TT.tknz = FALSE, 
                     lang = "fr", TT.options = list(path = "treetagger", preset = "fr-utf8"))
    dictionnaire <- rbind(dictionnaire, [email protected] )
  }
  return(unique(dictionnaire))
}

В този момент имам tdm с нещо подобно

                                                 Docs
Terms                                              Urbain.txt Versele.txt
  sudest                                                    0           1
  suit                                                      0           0
  suivi                                                     0           0
  sujets                                                    0           0
  supplémentaire                                            0           0
  suzanne                                                   0           0
  symbols                                                   0           0
  tant                                                      0           0
  tdm                                                       0           0
  télévisés                                                 0           0
  tempérament                                               0           0
  temps                                                     1           0
  termdocumentmatrixcorp                                    0           0
  terms                                                     0           0
  terre                                                     0           0
  tête                                                      0           0
  text                                                      0           0
  textcat                                                   0           0
  the                                                       0           1
  théâtre                                                   0           0
  thème                                                     0           0
  themebw                                                   0           0
  thérapeute                                                0           0
  thérapie                                                  0           0
  thèse                                                     0           0
  tissent                                                   0           0
  tmmapcorp                                                 0           0
  tmmapmytxt                                                0           0
  tokyo                                                     0           0
  tôt                                                       0           0
  touchent                                                  0           0
  toujours                                                  0           0
  tournant                                                  0           0
  tous                                                      0           0
  tout                                                      0           0
  toute                                                     0           0
  toutes                                                    0           0
  traditionnelle                                            0           1
  transformé                                                0           0
  travail                                                   0           0
  travaillant                                               0           1
  travaille                                                 0           0
  travaillé                                                 0           0
  travaillent                                               0           0

Сега бих искал да събера броя на думите с моя речник на lemme, нещо за групиране на travaillé, travaille, travaillant, travaillent...

в резултат на моята лематизация на функцията имам:

my.lemma[my.lemma$lemma == "travailler",]
           token      tag      lemma lttr wclass                    desc stop stem
665    travaillé VER:pper travailler    9   verb    verb past participle   NA   NA
835    travaille VER:pres travailler    9   verb            verb present   NA   NA
1369 travaillent VER:pres travailler   11   verb            verb present   NA   NA
1713 travaillant VER:ppre travailler   11   verb verb present participle   NA   NA

Не знам как да пристъпя към това събиране


r aggregate tm treetagger
person delaye    schedule 13.11.2015    source източник

Отговори (2)


arrow_upward
1
arrow_downward

Можеш да опиташ

aggregate(.~lemma, merge(tdm, mylemma[, c("token", "lemma")], by.x="row.names", by.y="token")[-1], sum)

което трябва да ви даде нещо подобно

#        lemma Urbain.txt Versele.txt
# 1 travailler          0           1
# ...
person lukeA    schedule 13.11.2015

arrow_upward
0
arrow_downward

Друг начин е да използвате dplyr.

Поставих целия код в R функция: lemmatization.tdm

corpus е обект от tm::Corpus, lang е езиковият параметър (вижте help(koRpus::kRp.POS.tags) за наличните езици) и treetaggerfilepath са файловете, където сте инсталирали treetagger (вижте).

lemmatization.tdm <- function( corpus, lang = "fr",
                           treetaggerfilepath = "~/Programs/treetagger/" )
{
# get packages
  require( koRpus ) ;
  require( tm ) ;
  require( dplyr ) ;

# run treetager
  dictionnaire <- data.frame()
  for(i in 1:length(corpus) )
  {
    lemma <- treetag(corpus[[i]][[1]], 
                     treetagger = "manual", 
                     format = "obj", TT.tknz = FALSE, 
                     lang = lang, 
                     TT.options = list(path = treetaggerfilepath, 
                                       preset = lang )
                     )
    dictionnaire <- rbind(dictionnaire, [email protected] )
  } ;
# select outpu from treetagger (token are originals words and lemma the lemmas)
  dictionnaire[, c("token", "lemma")]  -> dictionnaire ;
 # treetagger give sometimes more than one ouput, you need select one.
 dictionnaire %>% select(., token) %>% unique() %>% row.names() -> rownames1 ;
  dictionnaire[ as.numeric( rownames1 ), ] -> dictionnaire ;

# prepare classic tdm
  TermDocumentMatrix(corpus ) %>% 
    as.matrix(.) %>%
    as.data.frame(., stringsAsFactors = FALSE )  -> 
    tdm ;
# bind terms in data
  cbind( token = rownames( tdm ), tdm ) ->
    tdm ;

# bind dictionnaire and tdm on terms (token)
  right_join( dictionnaire, tdm,  
         by = "token" ) -> tdm.lemma ;

# agreggate count for lemmas
  tdm.lemma[,-1] %>% 
    as.data.frame(.) %>%
    group_by(., lemma) %>%
    summarise_all(., funs( sum( na_if(., 0L), na.rm = TRUE ) ) ) -> 
    tdm.lemma1 ;

#prepare output
  rownames( tdm.lemma1 ) <- tdm.lemma1$lemma  ;

  as.matrix( tdm.lemma1[, -1] ) -> tdm.lemma1 ;

  return( tdm.lemma1 ) ;
} ;

# return a tdm in matrix class. 

tdm %>% rowSums %>% sort(., decreasing = TRUE) -> countofterm # to see results by terms  

countofterm %>% sum -> numberofanalyzedwords
countofterm / numberofanalyzedwords -> freqterm

freqterm %>% filter(., freqterm > 0.1) # get words with a frequency above 0.1
person François Jean    schedule 11.08.2017