Pilot: Ultimate

Define Login Credentials

For writing to InfluxDBCloud you need to run usethis::edit_r_environ() and define at least the following environment variables:

###############################################################################
### For InfluxDB-Cloud
###############################################################################
ULTIMATE_INFLUXDB_URL = "<ultimate_influxdbcloud_url>"
# InfluxDB API-Token with Write for bucket "ultimate"
ULTIMATE_INFLUXDB_TOKEN = "<ultimate_influxdbcloud_apitoken>"
ULTIMATE_INFLUXDB_ORG = "<ultimate_influxdbcloud_organisation>"

################################################################################  
## Optional (for downloading raw data from Nextcloud)
################################################################################
NEXTCLOUD_URL = "https://<replace-with-nextcloud-cloud-url>"
NEXTCLOUD_USER = "<your-nextcloud-username>" # your username
NEXTCLOUD_PASSWORD = "your-nextcloud-app-password" ### see details below

#3 For creating <your-nextcloud-app-password>:
#3.1 go to: https://replace-with-nextcloud-url/index.php/settings/user/security
#3.2 scroll down to create new app password
#3.3 select a name e.g. r-script and copy the token and replace your-nextcloud-app-password

Finally you need to restart Rstudio and proceed with the code below:

Install R package

# Enable repository from kwb-r
options(repos = c(
  kwbr = 'https://kwb-r.r-universe.dev',
  CRAN = 'https://cloud.r-project.org'
))

# Download and install kwb.pilot in R
install.packages('kwb.pilot')

Define Paths

library(kwb.pilot)

paths_list <- list(
  root_ci = tempdir(),
  root_windows = "C:/kwb",
  root = "",
  common_root = "projects/ultimate/raw_data_pilots",
  data_raw = "data", 
  data_imp = "data_imported",
  a_site_code = "Pilot_A",
  b_site_code = "Pilot_B",
  a_cloud = "<common_root>/<a_site_code>",
  a_cloud_raw = "<a_cloud>/<data_raw>", 
  a_cloud_imported = "<a_cloud>/<data_imp>",
  a_local = "<root>/<a_cloud>",
  a_local_raw = "<a_local>/<data_raw>", 
  a_local_imported = "<a_local>/<data_imp>",
  b_cloud = "<common_root>/<b_site_code>",
  b_cloud_raw = "<b_cloud>/<data_raw>", 
  b_cloud_imported = "<b_cloud>/<data_imp>",
  b_local = "<root>/<b_cloud>",
  b_local_raw = "<b_local>/<data_raw>", 
  b_local_imported = "<b_local>/<data_imp>"
)

paths <- kwb.utils::resolve(
  paths_list, 
  root = ifelse(
    identical(Sys.getenv("CI"), "true"),
    paths_list$root_ci, 
    paths_list$root_windows
  )
)

Write to InfluxDBCloud

################################################################################
### 1. Write Pilot Plant Raw Data to InfluxDB Cloud
################################################################################

################################################################################
### 1.1 Pilot A
################################################################################

## Step: kwb.pilot::download_nextcloud_files() is optional and can be skipped
## if the data is provided in the `paths$raw_data_dir`
files_pilot_a <- kwb.pilot::download_nextcloud_files(
  dir_cloud = paths$a_cloud_raw,
  dir_local = paths$a_local_raw
)

if (length(files_pilot_a) > 0L) {
  
  tsv_paths <- list.files(
    path = paths$a_local_raw,
    full.names = TRUE,
    pattern = "xls$"
  )
  
  kwb.pilot::write_to_influxdb_loop(
    tsv_paths = tsv_paths,
    paths = list(
      site_code = paths$a_site_code,
      raw_data_dir = paths$a_local_raw
    ),
    changed_only = FALSE,
    max_tsv_files = 10,
    batch_size = 5000
  )
  
  ### Move all files for Pilot A`s cloud "raw" data directory to the "imported" 
  ### directory (in case that a file is already existing there: overwrite it)!
  paths_pilot_a <- data.frame(
    raw = file.path(paths$a_cloud_raw, files_pilot_a),
    imported = file.path(paths$a_cloud_imported, files_pilot_a
    )
  )
  
  kwb.pilot::move_nextcloud_files(paths_pilot_a, overwrite = TRUE)
  
} else {
  
  message(sprintf(
    "Nothing to do for '%s'! No new files in '%s'",
    paths$a_site_code,
    paths$a_cloud_raw
  ))
}

################################################################################
### 1.2 Pilot B
################################################################################

## Step: kwb.pilot::download_nextcloud_files() is optional and can be skipped
## if the data is provided in the `paths$raw_data_dir`
files_pilot_b <- kwb.pilot::download_nextcloud_files(
  dir_cloud = paths$b_cloud_raw,
  dir_local = paths$b_local_raw
)

if (length(files_pilot_b) > 0L) {
  
  tsv_paths <- list.files(
    path = paths$b_local_raw,
    full.names = TRUE,
    pattern = "xls$"
  )
  
  kwb.pilot::write_to_influxdb_loop(
    tsv_paths = tsv_paths,
    paths = list(
      site_code = paths$b_site_code,
      raw_data_dir = paths$b_local_raw
    ),
    changed_only = FALSE,
    max_tsv_files = 5,
    batch_size = 5000
  )
  
  ### Move all files for Pilot B`s cloud "raw" data directory to the "imported" 
  ### directory (in case that a file is already existing there: overwrite it)!
  paths_pilot_b <- data.frame(
    raw = file.path(paths$b_cloud_raw, files_pilot_b),
    imported = file.path(paths$b_cloud_imported, files_pilot_b)
  )
  
  kwb.pilot::move_nextcloud_files(paths_pilot_b, overwrite = TRUE)
  
} else {
  
  message(sprintf(
    "Nothing to do for '%s'! No new files in '%s'",
    paths$b_site_code,
    paths$b_cloud_raw
  ))
}