{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0c5d8b1f",
   "metadata": {},
   "source": [
    "# Tutorial 1 - Finding representations for new samples\n",
    "\n",
    "This tutorial shows how to find the best representations in latent space for a new set of samples, using a pre-trained bulkdgd model. We start from a CSV file of raw gene expression samples and end up with their latent representations and the decoder's predicted gene expression for each of them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "444642ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-05T18:10:38.645893Z",
     "iopub.status.busy": "2026-07-05T18:10:38.645402Z",
     "iopub.status.idle": "2026-07-05T18:10:43.617790Z",
     "shell.execute_reply": "2026-07-05T18:10:43.616482Z"
    }
   },
   "outputs": [],
   "source": [
    "# Import from the standard library.\n",
    "import logging as log\n",
    "\n",
    "# Import from third-party libraries.\n",
    "import pandas as pd\n",
    "\n",
    "# Import from 'bulkdgd'.\n",
    "from bulkdgd import ioutil\n",
    "from bulkdgd.core import model\n",
    "\n",
    "# Set the logging options so that every message of level INFO or above\n",
    "# is emitted.\n",
    "log.basicConfig(level = \"INFO\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "897c809b",
   "metadata": {},
   "source": [
    "## Preprocess the samples\n",
    "\n",
    "Load the raw samples from a CSV file, then preprocess them so that their genes match the set of genes included in the bulkdgd model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c307b979",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-05T18:10:43.622759Z",
     "iopub.status.busy": "2026-07-05T18:10:43.622418Z",
     "iopub.status.idle": "2026-07-05T18:10:48.507458Z",
     "shell.execute_reply": "2026-07-05T18:10:48.506117Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:Since 'keep_samples_names = True', the samples will be identified using the values contained in the first column of the data frame.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:16883 column(s) containing gene expression data was (were) found in the input data frame.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:16883 column(s) containing gene expression data was (were) found in the input data frame.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:1 column(s) containing additional information (not gene expression data) was (were) in the input data frame: tissue.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:Now looking for duplicated samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:No duplicated samples were found.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:Now looking for missing values in the columns containing gene expression data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:No missing values were found in the columns containing gene expression data.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:Now looking for duplicated genes...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:No duplicated genes were found.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:In the data frame containing the pre-processed samples, the columns containing gene expression data will be ordered according to the list of genes included in the DGD model (taken from '/home/lcb518/software/bulkdgd-2.0.0/bulkdgd/data/model/genes/genes.txt').\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.ioutil.samplesio:In the data frame containing the pre-processed samples, the columns found in the input data frame which did not contain gene expression data, if any were present, will be appended as the last columns of the data frame and appear in the same order as they did in the input data frame.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:bulkdgd.ioutil.samplesio:2964 gene(s) found in the input samples is (are) not part of the set of genes included in the DGD model. It (they) will be removed from the data frame of preprocessed samples.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:bulkdgd.ioutil.samplesio:976 gene(s) in the set of genes included in the DGD model was (were) not found in the input samples. A default count of 0 will be assigned to it (them) in all preprocessed samples.\n"
     ]
    }
   ],
   "source": [
    "# Load the samples into a data frame.\n",
    "df_samples = \\\n",
    "    ioutil.load_samples(# The CSV file where the samples are stored\n",
    "                        csv_file = \"samples.csv\",\n",
    "                        # The field separator used in the CSV file\n",
    "                        sep = \",\",\n",
    "                        # Whether to keep the original samples' names/\n",
    "                        # indexes (if True, they are assumed to be in\n",
    "                        # the first column of the data frame)\n",
    "                        keep_samples_names = True,\n",
    "                        # Whether to split the input data frame into\n",
    "                        # two data frames, one containing only gene\n",
    "                        # expression data and the other containing\n",
    "                        # additional information about the samples                  \n",
    "                        split = False)\n",
    "\n",
    "# Pre-process the samples.\n",
    "df_preproc, genes_excluded, genes_missing = \\\n",
    "    ioutil.preprocess_samples(df_samples = df_samples)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eeaa69de",
   "metadata": {},
   "source": [
    "## Load the configurations\n",
    "\n",
    "Load the configuration describing the pre-trained model's architecture and the default configuration for the two-round optimization scheme used to find representations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f02c7d76",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-05T18:10:48.510203Z",
     "iopub.status.busy": "2026-07-05T18:10:48.510030Z",
     "iopub.status.idle": "2026-07-05T18:10:48.522185Z",
     "shell.execute_reply": "2026-07-05T18:10:48.521212Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:bulkdgd.ioutil.configio:The configuration loaded from '/home/lcb518/software/bulkdgd-2.0.0/bulkdgd/configs/model/model_tgmm_trained.yaml' has warnings. Warnings: latent_options.random_state: no 'random_state' found. The default value 'None' will be used.|latent_options.cem: no 'cem' found. The default value 'False' will be used.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:bulkdgd.ioutil.configio:The configuration loaded from '/home/lcb518/software/bulkdgd-2.0.0/bulkdgd/configs/representations/two_opt.yaml' has warnings. Warnings: data_loader_options.shuffle: no 'shuffle' found. The default value 'False' will be used.|reporting_options.loss.reduction_type: no 'reduction_type' found. The default value 'sum' will be used.|reporting_options.loss.latent.lambda: no 'lambda' found. The default value '1.0' will be used.|scheme_options.loss_reduction_type: no 'loss_reduction_type' found. The default value 'sum' will be used.|scheme_options.latent_loss_calculation.lambda: no 'lambda' found. The default value '1.0' will be used.|scheme_options.optimization_1.auto_lr: no 'auto_lr' found. The default value 'False' will be used.|scheme_options.optimization_1.optimizer_type: no 'optimizer_type' found. The default value 'adamw' will be used.|scheme_options.optimization_2.auto_lr: no 'auto_lr' found. The default value 'False' will be used.|scheme_options.optimization_2.optimizer_type: no 'optimizer_type' found. The default value 'adamw' will be used.\n"
     ]
    }
   ],
   "source": [
    "# Load the configuration for the trained model.\n",
    "config_model = ioutil.load_config_model(\"model_tgmm_trained\")\n",
    "\n",
    "# Load the default configuration to find the best representations.\n",
    "config_rep = ioutil.load_config_rep(None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92c6bb20",
   "metadata": {},
   "source": [
    "## Get the trained model\n",
    "\n",
    "Instantiate the bulkdgd model (Gaussian mixture latent space and decoder), loading the pre-trained parameters referenced in the model configuration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7b8150ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-05T18:10:48.525191Z",
     "iopub.status.busy": "2026-07-05T18:10:48.525023Z",
     "iopub.status.idle": "2026-07-05T18:10:49.898936Z",
     "shell.execute_reply": "2026-07-05T18:10:49.898099Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:The parameters were successfully loaded from '/home/lcb518/software/bulkdgd-2.0.0/bulkdgd/data/model/gmm/gmm.pth'.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:The latent space was successfully set (type: 'GaussianMixtureModelTGMM').\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.decoders:The decoder's hidden layer # 1 was set. Input features: 32. Output features: 500. Activation function: 'ReLU'.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.decoders:The decoder's hidden layer # 2 was set. Input features: 500. Output features: 8000. Activation function: 'ReLU'.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.decoders:The decoder's output module was set. Module 'OutputModuleNBFeatureDispersion' (activation = 'softplus', r_init = 2, output_dim = 14895).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.decoders:The decoder's dropout fraction was set to 0.0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:The parameters were successfully loaded from '/home/lcb518/software/bulkdgd-2.0.0/bulkdgd/data/model/dec/dec.pth'.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:The decoder was successfully set.\n"
     ]
    }
   ],
   "source": [
    "# Get the trained bulkDGD model (latent space and decoder).\n",
    "dgd_model = model.BulkDGD(**config_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bd48bcc",
   "metadata": {},
   "source": [
    "## Get the representations\n",
    "\n",
    "For each preprocessed sample, find the representation in latent space that best reconstructs its gene expression profile, along with the decoder's predicted means (and r-values, if applicable) for that representation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd756909",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-05T18:10:49.900643Z",
     "iopub.status.busy": "2026-07-05T18:10:49.900484Z",
     "iopub.status.idle": "2026-07-05T18:11:49.678904Z",
     "shell.execute_reply": "2026-07-05T18:11:49.677435Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Starting optimization number 1...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 1: loss 45603980.847, epoch CPU time 94.978 s, backward step CPU time 69.156 s, epoch wall clock time 1.993 s, backward step wall clock time 1.567 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 2: loss 42951241.813, epoch CPU time 63.767 s, backward step CPU time 46.601 s, epoch wall clock time 1.269 s, backward step wall clock time 0.994 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 3: loss 40717904.042, epoch CPU time 65.341 s, backward step CPU time 47.581 s, epoch wall clock time 1.289 s, backward step wall clock time 1.004 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 4: loss 38804133.988, epoch CPU time 63.579 s, backward step CPU time 45.744 s, epoch wall clock time 1.233 s, backward step wall clock time 0.941 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 5: loss 37168256.945, epoch CPU time 63.162 s, backward step CPU time 45.644 s, epoch wall clock time 1.215 s, backward step wall clock time 0.934 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 6: loss 35750095.249, epoch CPU time 64.144 s, backward step CPU time 46.684 s, epoch wall clock time 1.244 s, backward step wall clock time 0.965 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 7: loss 34495569.898, epoch CPU time 66.626 s, backward step CPU time 48.442 s, epoch wall clock time 1.276 s, backward step wall clock time 0.982 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 8: loss 33387390.296, epoch CPU time 63.840 s, backward step CPU time 46.356 s, epoch wall clock time 1.215 s, backward step wall clock time 0.932 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 9: loss 32403827.989, epoch CPU time 63.980 s, backward step CPU time 46.545 s, epoch wall clock time 1.233 s, backward step wall clock time 0.947 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 10: loss 31490056.736, epoch CPU time 64.093 s, backward step CPU time 46.492 s, epoch wall clock time 1.230 s, backward step wall clock time 0.937 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Starting optimization number 2...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 1: loss 552285.162, epoch CPU time 38.298 s, backward step CPU time 32.524 s, epoch wall clock time 0.795 s, backward step wall clock time 0.701 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 2: loss 549961.641, epoch CPU time 38.409 s, backward step CPU time 32.598 s, epoch wall clock time 0.778 s, backward step wall clock time 0.686 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 3: loss 547966.925, epoch CPU time 38.068 s, backward step CPU time 32.315 s, epoch wall clock time 0.774 s, backward step wall clock time 0.684 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 4: loss 546150.016, epoch CPU time 38.254 s, backward step CPU time 32.369 s, epoch wall clock time 0.777 s, backward step wall clock time 0.685 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 5: loss 544545.439, epoch CPU time 38.058 s, backward step CPU time 32.236 s, epoch wall clock time 0.789 s, backward step wall clock time 0.697 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 6: loss 543099.304, epoch CPU time 38.360 s, backward step CPU time 32.474 s, epoch wall clock time 0.800 s, backward step wall clock time 0.707 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 7: loss 541808.775, epoch CPU time 40.938 s, backward step CPU time 34.865 s, epoch wall clock time 0.991 s, backward step wall clock time 0.896 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 8: loss 540644.184, epoch CPU time 38.729 s, backward step CPU time 32.473 s, epoch wall clock time 0.808 s, backward step wall clock time 0.706 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 9: loss 539530.130, epoch CPU time 45.413 s, backward step CPU time 38.649 s, epoch wall clock time 1.029 s, backward step wall clock time 0.916 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 10: loss 538532.102, epoch CPU time 44.142 s, backward step CPU time 37.708 s, epoch wall clock time 1.022 s, backward step wall clock time 0.913 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 11: loss 537556.810, epoch CPU time 96.729 s, backward step CPU time 53.782 s, epoch wall clock time 1.974 s, backward step wall clock time 1.176 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 12: loss 536631.636, epoch CPU time 39.542 s, backward step CPU time 33.467 s, epoch wall clock time 0.826 s, backward step wall clock time 0.731 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 13: loss 535780.297, epoch CPU time 39.583 s, backward step CPU time 33.569 s, epoch wall clock time 0.833 s, backward step wall clock time 0.739 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 14: loss 535001.137, epoch CPU time 38.843 s, backward step CPU time 32.957 s, epoch wall clock time 0.806 s, backward step wall clock time 0.713 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 15: loss 534276.568, epoch CPU time 38.137 s, backward step CPU time 32.507 s, epoch wall clock time 0.779 s, backward step wall clock time 0.689 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 16: loss 533611.648, epoch CPU time 48.465 s, backward step CPU time 37.903 s, epoch wall clock time 1.034 s, backward step wall clock time 0.843 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 17: loss 532973.171, epoch CPU time 38.346 s, backward step CPU time 32.581 s, epoch wall clock time 0.809 s, backward step wall clock time 0.718 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 18: loss 532366.687, epoch CPU time 38.412 s, backward step CPU time 32.523 s, epoch wall clock time 0.826 s, backward step wall clock time 0.735 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 19: loss 531801.833, epoch CPU time 38.544 s, backward step CPU time 32.601 s, epoch wall clock time 0.766 s, backward step wall clock time 0.673 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 20: loss 531272.190, epoch CPU time 38.920 s, backward step CPU time 32.965 s, epoch wall clock time 0.858 s, backward step wall clock time 0.766 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 21: loss 530756.620, epoch CPU time 58.289 s, backward step CPU time 52.405 s, epoch wall clock time 1.150 s, backward step wall clock time 1.054 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 22: loss 530274.893, epoch CPU time 38.463 s, backward step CPU time 32.511 s, epoch wall clock time 0.822 s, backward step wall clock time 0.728 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 23: loss 529834.029, epoch CPU time 38.985 s, backward step CPU time 32.964 s, epoch wall clock time 0.831 s, backward step wall clock time 0.736 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 24: loss 529421.617, epoch CPU time 39.895 s, backward step CPU time 34.086 s, epoch wall clock time 0.850 s, backward step wall clock time 0.760 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 25: loss 529024.665, epoch CPU time 38.160 s, backward step CPU time 32.461 s, epoch wall clock time 0.813 s, backward step wall clock time 0.722 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 26: loss 528647.507, epoch CPU time 37.850 s, backward step CPU time 32.294 s, epoch wall clock time 0.811 s, backward step wall clock time 0.723 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 27: loss 528274.480, epoch CPU time 37.744 s, backward step CPU time 31.985 s, epoch wall clock time 0.784 s, backward step wall clock time 0.695 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 28: loss 527912.499, epoch CPU time 38.791 s, backward step CPU time 33.218 s, epoch wall clock time 0.832 s, backward step wall clock time 0.745 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 29: loss 527566.968, epoch CPU time 37.902 s, backward step CPU time 32.324 s, epoch wall clock time 0.809 s, backward step wall clock time 0.719 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 30: loss 527228.718, epoch CPU time 37.385 s, backward step CPU time 31.936 s, epoch wall clock time 0.776 s, backward step wall clock time 0.690 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 31: loss 526912.822, epoch CPU time 38.474 s, backward step CPU time 32.661 s, epoch wall clock time 0.838 s, backward step wall clock time 0.745 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 32: loss 526619.766, epoch CPU time 38.511 s, backward step CPU time 32.623 s, epoch wall clock time 0.808 s, backward step wall clock time 0.716 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 33: loss 526339.122, epoch CPU time 37.974 s, backward step CPU time 32.213 s, epoch wall clock time 0.778 s, backward step wall clock time 0.687 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 34: loss 526078.573, epoch CPU time 38.388 s, backward step CPU time 32.521 s, epoch wall clock time 0.792 s, backward step wall clock time 0.698 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 35: loss 525827.510, epoch CPU time 38.182 s, backward step CPU time 32.429 s, epoch wall clock time 0.802 s, backward step wall clock time 0.711 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 36: loss 525608.569, epoch CPU time 38.053 s, backward step CPU time 32.285 s, epoch wall clock time 0.811 s, backward step wall clock time 0.721 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 37: loss 525414.128, epoch CPU time 38.711 s, backward step CPU time 32.823 s, epoch wall clock time 0.811 s, backward step wall clock time 0.718 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 38: loss 525228.375, epoch CPU time 37.799 s, backward step CPU time 31.920 s, epoch wall clock time 0.768 s, backward step wall clock time 0.674 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 39: loss 525057.508, epoch CPU time 37.282 s, backward step CPU time 31.643 s, epoch wall clock time 0.755 s, backward step wall clock time 0.669 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 40: loss 524902.103, epoch CPU time 37.296 s, backward step CPU time 31.378 s, epoch wall clock time 0.747 s, backward step wall clock time 0.651 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 41: loss 524755.774, epoch CPU time 43.395 s, backward step CPU time 37.755 s, epoch wall clock time 0.843 s, backward step wall clock time 0.755 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 42: loss 524612.329, epoch CPU time 36.934 s, backward step CPU time 31.480 s, epoch wall clock time 0.739 s, backward step wall clock time 0.653 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 43: loss 524485.540, epoch CPU time 36.959 s, backward step CPU time 31.382 s, epoch wall clock time 0.754 s, backward step wall clock time 0.668 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 44: loss 524361.854, epoch CPU time 37.133 s, backward step CPU time 31.653 s, epoch wall clock time 0.773 s, backward step wall clock time 0.686 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 45: loss 524253.516, epoch CPU time 37.308 s, backward step CPU time 31.924 s, epoch wall clock time 0.779 s, backward step wall clock time 0.693 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 46: loss 524156.123, epoch CPU time 37.864 s, backward step CPU time 32.168 s, epoch wall clock time 0.799 s, backward step wall clock time 0.708 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 47: loss 524059.030, epoch CPU time 37.329 s, backward step CPU time 31.559 s, epoch wall clock time 0.782 s, backward step wall clock time 0.691 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 48: loss 523960.465, epoch CPU time 37.103 s, backward step CPU time 31.402 s, epoch wall clock time 0.778 s, backward step wall clock time 0.689 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 49: loss 523863.397, epoch CPU time 36.978 s, backward step CPU time 31.319 s, epoch wall clock time 0.763 s, backward step wall clock time 0.676 s.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bulkdgd.core.model:Epoch 50: loss 523773.521, epoch CPU time 36.867 s, backward step CPU time 31.238 s, epoch wall clock time 0.765 s, backward step wall clock time 0.676 s.\n"
     ]
    }
   ],
   "source": [
    "# Get the representations, the predicted scaled means and\n",
    "# r-values of the negative binomials modeling the genes' counts for\n",
    "# the in silico samples corresponding to the representations found,\n",
    "# and the time spent finding the representations.\n",
    "df_rep, df_pred_means, df_pred_r_values, df_time_opt = \\\n",
    "    dgd_model.get_representations(\\\n",
    "        # The data frame with the pre-processed samples\n",
    "        df_samples = df_preproc,\n",
    "        # The configuration to find the representations\n",
    "        config_rep = config_rep)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e7c118b",
   "metadata": {},
   "source": [
    "## Save the outputs\n",
    "\n",
    "Save the preprocessed samples, the representations found, the decoder's predictions, and the time spent on the optimization to CSV files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "edcbf8ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-07-05T18:11:49.684189Z",
     "iopub.status.busy": "2026-07-05T18:11:49.683775Z",
     "iopub.status.idle": "2026-07-05T18:11:54.839245Z",
     "shell.execute_reply": "2026-07-05T18:11:54.837501Z"
    }
   },
   "outputs": [],
   "source": [
    "# Save the pre-processed samples.\n",
    "ioutil.save_samples(\\\n",
    "   # The data frame containing the samples\n",
    "   df = df_preproc,\n",
    "   # The output CSV file\n",
    "   csv_file = \"samples_preprocessed.csv\",\n",
    "   # The field separator in the output CSV file\n",
    "   sep = \",\")\n",
    "\n",
    "# Save the representations.\n",
    "ioutil.save_representations(\\\n",
    "    # The data frame containing the representations\n",
    "    df = df_rep,\n",
    "    # The output CSV file\n",
    "    csv_file = \"representations.csv\",\n",
    "    # The field separator in the output CSV file\n",
    "    sep = \",\")\n",
    "\n",
    "# Save the predicted scaled means.\n",
    "ioutil.save_decoder_outputs(\\\n",
    "    # The data frame containing the predicted scaled means\n",
    "    df = df_pred_means,\n",
    "    # The output CSV file\n",
    "    csv_file = \"pred_means.csv\",\n",
    "    # The field separator in the output CSV file\n",
    "    sep = \",\")\n",
    "\n",
    "# Save the predicted r-values.\n",
    "ioutil.save_decoder_outputs(\\\n",
    "    # The data frame containing the predicted r-values\n",
    "    df = df_pred_r_values,\n",
    "    # The output CSV file\n",
    "    csv_file = \"pred_r_values.csv\",\n",
    "    # The field separator in the output CSV file\n",
    "    sep = \",\")\n",
    "\n",
    "# Save the time data.\n",
    "df_time_opt.to_csv(\"time_opt.csv\",\n",
    "                   index = True,\n",
    "                   header = True,\n",
    "                   sep = \",\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bulkdgd2-env",
   "language": "python",
   "name": "bulkdgd2-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}