R combining duplicate rows in a time series with different column types in a datatable
We can do a group by mean
library(data.table)
library(tidyr)
library(dplyr)
dt[, lapply(.SD, function(x) if(is.numeric(x)) mean(x, na.rm = TRUE)
else toString(unique(x[!is.na(x)]))), .(time)] %>%
separate_rows(farbe, goff)
# A tibble: 7 x 7
# time abst farbe gier goff huft mode
# <dbl> <dbl> <chr> <dbl> <chr> <chr> <dbl>
#1 1 1 keine 2.5 "haus" "" 4
#2 1 1 keine 2.5 "maus" "" 4
#3 1 1 keine 2.5 "toll" "" 4
#4 2 0 keine 0 "maus" "wolle" 2.5
#5 3 0 keine 0 "" "wolle" NaN
#6 4 3 rot 3 "maus" "holz" 3
#7 4 3 blau 3 "maus" "holz" 3
Or with cSplit
library(splitstackshape)
cSplit(dt[, lapply(.SD, function(x) if(is.numeric(x))
mean(x, na.rm = TRUE) else toString(unique(x[!is.na(x)]))), .(time)],
c('farbe', 'goff'), sep= ',\\s*', 'long', fixed = FALSE)
# time abst farbe gier goff huft mode
#1: 1 1 keine 2.5 haus 4.0
#2: 1 1 <NA> 2.5 maus 4.0
#3: 1 1 <NA> 2.5 toll 4.0
#4: 2 0 keine 0.0 maus wolle 2.5
#5: 3 0 keine 0.0 <NA> wolle NaN
#6: 4 3 rot 3.0 maus holz 3.0
#7: 4 3 blau 3.0 <NA> holz 3.0
The expected result (for the given sample dataset) can also be achieved without a subsequent call to separate_rows()
or cSplit()
:
library(data.table) # version 1.12.9
dt[, lapply(.SD, function(x) if (is.numeric(x)) mean(x, na.rm = TRUE)
else unlist(na.omit(unique(x)))), by = time]
time abst farbe gier goff huft mode
1: 1 1 keine 2.5 haus <NA> 4.0
2: 1 1 keine 2.5 maus <NA> 4.0
3: 1 1 keine 2.5 toll <NA> 4.0
4: 2 0 keine 0.0 maus wolle 2.5
5: 3 0 keine 0.0 <NA> wolle NaN
6: 4 3 rot 3.0 maus holz 3.0
7: 4 3 blau 3.0 maus holz 3.0
Please, note that this approach will work for an arbitrary mix of numeric and factor columns; no column names need to be stated explicitly.
However, I do believe the correct answer to the underlying problem is to return one row per time
instead of a kind of partial aggregate (your mileage may vary, of course):
dt[, lapply(.SD, function(x) if (is.numeric(x)) mean(x, na.rm = TRUE)
else list(na.omit(unique(x)))), by = time]
time abst farbe gier goff huft mode 1: 1 1 keine 2.5 haus,maus,toll 4.0 2: 2 0 keine 0.0 maus wolle 2.5 3: 3 0 keine 0.0 wolle NaN 4: 4 3 rot,blau 3.0 maus holz 3.0
Please, note that list()
instead of toString()
has been used to aggregate the factor columns. This has the benefit to avoid problems in case one of the factor levels includes a comma ,
by chance. Furthermore, it is easier to identify cases with non-unique factors per time
in a large production dataset:
# compute aggregate as before
dtRes <- dt[, lapply(.SD, function(x) if (is.numeric(x)) mean(x, na.rm = TRUE)
else list(na.omit(unique(x)))), by = time]
# find cases with non-unique factors per group
# note .SDcols = is.list is available with data.table version 1.12.9
tmp <- dtRes[, which(Reduce(sum, lapply(.SD, function(x) lengths(x) > 1L)) > 0), .SDcols = is.list, by = time]
tmp
time V1 1: 1 1 2: 4 1
# show affected rows
dtRes[tmp, on = "time"]
time abst farbe gier goff huft mode V1 1: 1 1 keine 2.5 haus,maus,toll 4 1 2: 4 3 rot,blau 3.0 maus holz 3 1
# show not affected rows
dtRes[!tmp, on = "time"]
time abst farbe gier goff huft mode 1: 2 0 keine 0 maus wolle 2.5 2: 3 0 keine 0 wolle NaN