SCM

SCM Repository

[tm] Diff of /pkg/R/corpus.R
ViewVC logotype

Diff of /pkg/R/corpus.R

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

trunk/R/trunk/R/textdoccol.R revision 18, Sat Nov 5 19:00:05 2005 UTC pkg/R/corpus.R revision 1377, Wed May 21 17:15:56 2014 UTC
# Line 1  Line 1 
1  # Author: Ingo Feinerer  # Author: Ingo Feinerer
2    
3  # S4 class definition  PCorpus <-
4  # Text document collection  function(x,
5  # TODO: Define proper S4 term-document matrix           readerControl = list(reader = reader(x), language = "en"),
6  setClass("textdoccol", representation(docs = "list",           dbControl = list(dbName = "", dbType = "DB1"))
7                                        tdm = "matrix"))  {
8        stopifnot(inherits(x, "Source"))
9  # Accessor function  
10  if (!isGeneric("docs")) {      readerControl <- prepareReader(readerControl, reader(x))
11      if (is.function("docs"))  
12          fun <- docs      if (is.function(readerControl$init))
13            readerControl$init()
14    
15        if (is.function(readerControl$exit))
16            on.exit(readerControl$exit())
17    
18        if (!filehash::dbCreate(dbControl$dbName, dbControl$dbType))
19            stop("error in creating database")
20        db <- filehash::dbInit(dbControl$dbName, dbControl$dbType)
21    
22        tdl <- vector("list", length(x))
23        counter <- 1
24        while (!eoi(x)) {
25            x <- stepNext(x)
26            elem <- getElem(x)
27            doc <- readerControl$reader(elem,
28                                        readerControl$language,
29                                        as.character(counter))
30            filehash::dbInsert(db, meta(doc, "id"), doc)
31            tdl[[counter]] <- meta(doc, "id")
32            counter <- counter + 1
33        }
34    
35        structure(list(content = tdl,
36                       meta = CorpusMeta(),
37                       dmeta = data.frame(row.names = seq_along(tdl)),
38                       dbcontrol = dbControl),
39                  class = c("PCorpus", "Corpus"))
40    }
41    
42    Corpus <-
43    VCorpus <-
44    function(x, readerControl = list(reader = reader(x), language = "en"))
45    {
46        stopifnot(inherits(x, "Source"))
47    
48        readerControl <- prepareReader(readerControl, reader(x))
49    
50        if (is.function(readerControl$init))
51            readerControl$init()
52    
53        if (is.function(readerControl$exit))
54            on.exit(readerControl$exit())
55    
56        tdl <- vector("list", length(x))
57        # Check for parallel element access
58        if (is.function(getS3method("pGetElem", class(x), TRUE)))
59            tdl <- mapply(function(elem, id)
60                              readerControl$reader(elem, readerControl$language, id),
61                          pGetElem(x),
62                          id = as.character(seq_along(x)),
63                          SIMPLIFY = FALSE)
64        else {
65            counter <- 1
66            while (!eoi(x)) {
67                x <- stepNext(x)
68                elem <- getElem(x)
69                doc <- readerControl$reader(elem,
70                                            readerControl$language,
71                                            as.character(counter))
72                tdl[[counter]] <- doc
73                counter <- counter + 1
74            }
75        }
76    
77        structure(list(content = tdl,
78                       meta = CorpusMeta(),
79                       dmeta = data.frame(row.names = seq_along(tdl))),
80                  class = c("VCorpus", "Corpus"))
81    }
82    
83    `[.PCorpus` <-
84    function(x, i)
85    {
86        if (!missing(i)) {
87            x$content <- x$content[i]
88            x$dmeta <- x$dmeta[i, , drop = FALSE]
89        }
90        x
91    }
92    
93    `[.VCorpus` <-
94    function(x, i)
95    {
96        if (!missing(i)) {
97            x$content <- x$content[i]
98            x$dmeta <- x$dmeta[i, , drop = FALSE]
99            if (!is.null(x$lazy))
100                x$lazy$index <- x$lazy$index[i]
101        }
102        x
103    }
104    
105    .map_name_index <-
106    function(x, i)
107    {
108        if (is.character(i))
109            match(i, meta(x, "id", "local"))
110      else      else
111          fun <- function(object) standardGeneric("docs")          i
112      setGeneric("docs", fun)  }
113    
114    `[[.PCorpus` <-
115    function(x, i)
116    {
117        i <- .map_name_index(x, i)
118        db <- filehash::dbInit(x$dbcontrol[["dbName"]], x$dbcontrol[["dbType"]])
119        filehash::dbFetch(db, x$content[[i]])
120    }
121    `[[.VCorpus` <-
122    function(x, i)
123    {
124        i <- .map_name_index(x, i)
125        if (!is.null(x$lazy))
126            .Call(copyCorpus, x, materialize(x, i))
127        x$content[[i]]
128  }  }
 setMethod("docs", "textdoccol", function(object) object@docs)  
129    
130  setGeneric("textdoccol", function(docs) standardGeneric("textdoccol"))  `[[<-.PCorpus` <-
131  # Read in XML text documents  function(x, i, value)
132  # Reuters Corpus Volume 1 (RCV1)  {
133  setMethod("textdoccol", "character", function(docs) {      i <- .map_name_index(x, i)
134      require(XML)      db <- filehash::dbInit(x$dbcontrol[["dbName"]], x$dbcontrol[["dbType"]])
135        db[[x$content[[i]]]] <- value
136      tree <- xmlTreeParse(docs)      x
137      root <- xmlRoot(tree)  }
138    `[[<-.VCorpus` <-
139      # TODO: At each loop node points to the current newsitem  function(x, i, value)
140      node <- root  {
141        i <- .map_name_index(x, i)
142      # TODO: Implement lacking fields.      # Mark new objects as inactive for lazy mapping
143      # For this we need the full RCV1 XML set to know where to find those things      if (!is.null(x$lazy))
144      author <- "Not yet implemented"          x$lazy$index[i] <- FALSE
145      timestamp <- xmlAttrs(node)[["date"]]      x$content[[i]] <- value
146      description <- "Not yet implemented"      x
147      id <- as.integer(xmlAttrs(node)[["itemid"]])  }
     origin <- "Not yet implemented"  
     corpus <- unlist(xmlApply(node[["text"]], xmlValue), use.names = FALSE)  
148    
149      heading <- xmlValue(node[["title"]])  as.list.PCorpus <- as.list.VCorpus <-
150    function(x, ...)
151        content(x)
152    
153    as.VCorpus <-
154    function(x)
155        UseMethod("as.VCorpus")
156    as.VCorpus.VCorpus <- identity
157    
158    outer_union <-
159    function(x, y, ...)
160    {
161        if (nrow(x) > 0L)
162            x[, setdiff(names(y), names(x))] <- NA
163        if (nrow(y) > 0L)
164            y[, setdiff(names(x), names(y))] <- NA
165        res <- rbind(x, y)
166        if (ncol(res) == 0L)
167            res <- data.frame(row.names = seq_len(nrow(x) + nrow(y)))
168        res
169    }
170    
171      doc <- new("textdocument", author = author, timestamp = timestamp, description = description,  c.VCorpus <-
172                 id = id, origin = origin, corpus = corpus, heading = heading)  function(..., recursive = FALSE)
173    {
174        args <- list(...)
175        x <- args[[1L]]
176    
177        if (length(args) == 1L)
178            return(x)
179    
180        if (!all(unlist(lapply(args, inherits, class(x)))))
181            stop("not all arguments are of the same corpus type")
182    
183        structure(list(content = do.call("c", lapply(args, content)),
184                       meta = CorpusMeta(meta = do.call("c",
185                         lapply(args, function(a) meta(a, type = "corpus")))),
186                       dmeta = Reduce(outer_union, lapply(args, meta))),
187                  class = c("VCorpus", "Corpus"))
188    }
189    
190    content.VCorpus <-
191    function(x)
192    {
193        if (!is.null(x$lazy))
194            .Call(copyCorpus, x, materialize(x))
195        x$content
196    }
197    
198    content.PCorpus <-
199    function(x)
200    {
201        db <- filehash::dbInit(x$dbcontrol[["dbName"]], x$dbcontrol[["dbType"]])
202        filehash::dbMultiFetch(db, unlist(x$content))
203    }
204    
205      new("textdoccol", docs = list(doc), tdm = matrix())  inspect <-
206  })  function(x)
207        UseMethod("inspect", x)
208    inspect.PCorpus <- inspect.VCorpus <-
209    function(x)
210    {
211        print(x)
212        cat("\n")
213        print(noquote(content(x)))
214        invisible(x)
215    }
216    
217    length.PCorpus <- length.VCorpus <-
218    function(x)
219        length(x$content)
220    
221    names.PCorpus <- names.VCorpus <-
222    function(x)
223        as.character(meta(x, "id", "local"))
224    
225    print.PCorpus <- print.VCorpus <-
226    function(x, ...)
227    {
228        writeLines(sprintf("<<%s (documents: %d, metadata (corpus/indexed): %d/%d)>>",
229                           class(x)[1],
230                           length(x),
231                           length(meta(x, type = "corpus")),
232                           ncol(meta(x, type = "indexed"))))
233        invisible(x)
234    }
235    
236    writeCorpus <-
237    function(x, path = ".", filenames = NULL)
238    {
239        filenames <- file.path(path,
240          if (is.null(filenames))
241              sprintf("%s.txt", as.character(meta(x, "id", "local")))
242          else filenames)
243    
244        stopifnot(length(x) == length(filenames))
245    
246        mapply(function(doc, f) writeLines(as.character(doc), f), x, filenames)
247    
248        invisible(x)
249    }

Legend:
Removed from v.18  
changed lines
  Added in v.1377

root@r-forge.r-project.org
ViewVC Help
Powered by ViewVC 1.0.0  
Thanks to:
Vienna University of Economics and Business Powered By FusionForge