jdti.jdti
1import math 2import os 3import pickle 4import re 5from typing import Literal 6 7import harmonypy as harmonize 8import matplotlib.pyplot as plt 9import numpy as np 10import pandas as pd 11import plotly.express as px 12import seaborn as sns 13import umap 14from adjustText import adjust_text 15from joblib import Parallel, delayed 16from matplotlib.lines import Line2D 17from matplotlib.patches import FancyArrowPatch, Patch, Polygon 18from scipy import sparse 19from scipy.io import mmwrite 20from scipy.spatial import ConvexHull 21from scipy.spatial.distance import pdist, squareform 22from scipy.stats import norm, stats, zscore 23from sklearn.cluster import DBSCAN, MeanShift 24from sklearn.decomposition import PCA 25from sklearn.metrics import pairwise_distances, silhouette_score 26from sklearn.preprocessing import StandardScaler 27from tqdm import tqdm 28 29from .utils import * 30 31 32class Clustering: 33 """ 34 A class for performing dimensionality reduction, clustering, and visualization 35 on high-dimensional data (e.g., single-cell gene expression). 36 37 The class provides methods for: 38 - Normalizing and extracting subsets of data 39 - Principal Component Analysis (PCA) and related clustering 40 - Uniform Manifold Approximation and Projection (UMAP) and clustering 41 - Visualization of PCA and UMAP embeddings 42 - Harmonization of batch effects 43 - Accessing processed data and cluster labels 44 45 Methods 46 ------- 47 add_data_frame(data, metadata) 48 Class method to create a Clustering instance from a DataFrame and metadata. 49 50 harmonize_sets() 51 Perform batch effect harmonization on PCA data. 52 53 perform_PCA(pc_num=100, width=8, height=6) 54 Perform PCA on the dataset and visualize the first two PCs. 55 56 knee_plot_PCA(width=8, height=6) 57 Plot the cumulative variance explained by PCs to determine optimal dimensionality. 58 59 find_clusters_PCA(pc_num=0, eps=0.5, min_samples=10, width=8, height=6, harmonized=False) 60 Apply DBSCAN clustering to PCA embeddings and visualize results. 61 62 perform_UMAP(factorize=False, umap_num=100, pc_num=0, harmonized=False, ...) 63 Compute UMAP embeddings with optional parameter tuning. 64 65 knee_plot_umap(eps=0.5, min_samples=10) 66 Determine optimal UMAP dimensionality using silhouette scores. 67 68 find_clusters_UMAP(umap_n=5, eps=0.5, min_samples=10, width=8, height=6) 69 Apply DBSCAN clustering on UMAP embeddings and visualize clusters. 70 71 UMAP_vis(names_slot='cell_names', set_sep=True, point_size=0.6, ...) 72 Visualize UMAP embeddings with labels and optional cluster numbering. 73 74 UMAP_feature(feature_name, features_data=None, point_size=0.6, ...) 75 Plot a single feature over UMAP coordinates with customizable colormap. 76 77 get_umap_data() 78 Return the UMAP embeddings along with cluster labels if available. 79 80 get_pca_data() 81 Return the PCA results along with cluster labels if available. 82 83 return_clusters(clusters='umap') 84 Return the cluster labels for UMAP or PCA embeddings. 85 86 Raises 87 ------ 88 ValueError 89 For invalid parameters, mismatched dimensions, or missing metadata. 90 """ 91 92 def __init__(self, data, metadata): 93 """ 94 Initialize the clustering class with data and optional metadata. 95 96 Parameters 97 ---------- 98 data : pandas.DataFrame 99 Input data for clustering. Columns are considered as samples. 100 101 metadata : pandas.DataFrame, optional 102 Metadata for the samples. If None, a default DataFrame with column 103 names as 'cell_names' is created. 104 105 Attributes 106 ---------- 107 -clustering_data : pandas.DataFrame 108 -clustering_metadata : pandas.DataFrame 109 -subclusters : None or dict 110 -explained_var : None or numpy.ndarray 111 -cumulative_var : None or numpy.ndarray 112 -pca : None or pandas.DataFrame 113 -harmonized_pca : None or pandas.DataFrame 114 -umap : None or pandas.DataFrame 115 """ 116 117 self.clustering_data = data 118 """The input data used for clustering.""" 119 120 if metadata is None: 121 metadata = pd.DataFrame({"cell_names": list(data.columns)}) 122 123 self.clustering_metadata = metadata 124 """Metadata associated with the samples.""" 125 126 self.subclusters = None 127 """Placeholder for storing subcluster information.""" 128 129 self.explained_var = None 130 """Explained variance from PCA, initialized as None.""" 131 132 self.cumulative_var = None 133 """Cumulative explained variance from PCA, initialized as None.""" 134 135 self.pca = None 136 """PCA-transformed data, initialized as None.""" 137 138 self.harmonized_pca = None 139 """PCA data after batch effect harmonization, initialized as None.""" 140 141 self.umap = None 142 """UMAP embeddings, initialized as None.""" 143 144 @classmethod 145 def add_data_frame(cls, data: pd.DataFrame, metadata: pd.DataFrame | None): 146 """ 147 Create a Clustering instance from a DataFrame and optional metadata. 148 149 Parameters 150 ---------- 151 data : pandas.DataFrame 152 Input data with features as rows and samples/cells as columns. 153 154 metadata : pandas.DataFrame or None 155 Optional metadata for the samples. 156 Each row corresponds to a sample/cell, and column names in this DataFrame 157 should match the sample/cell names in `data`. Columns can contain additional 158 information such as cell type, experimental condition, batch, sets, etc. 159 160 Returns 161 ------- 162 Clustering 163 A new instance of the Clustering class. 164 """ 165 166 return cls(data, metadata) 167 168 def harmonize_sets(self, batch_col: str = "sets"): 169 """ 170 Perform batch effect harmonization on PCA embeddings. 171 172 Parameters 173 ---------- 174 batch_col : str, default 'sets' 175 Name of the column in `metadata` that contains batch information for the samples/cells. 176 177 Returns 178 ------- 179 None 180 Updates the `harmonized_pca` attribute with harmonized data. 181 """ 182 183 data_mat = np.array(self.pca) 184 185 metadata = self.clustering_metadata 186 187 self.harmonized_pca = pd.DataFrame( 188 harmonize.run_harmony(data_mat, metadata, vars_use=batch_col).Z_corr 189 ).T 190 191 self.harmonized_pca.columns = self.pca.columns 192 193 def perform_PCA(self, pc_num: int = 100, width=8, height=6): 194 """ 195 Perform Principal Component Analysis (PCA) on the dataset. 196 197 This method standardizes the data, applies PCA, stores results as attributes, 198 and generates a scatter plot of the first two principal components. 199 200 Parameters 201 ---------- 202 pc_num : int, default 100 203 Number of principal components to compute. 204 If 0, computes all available components. 205 206 width : int or float, default 8 207 Width of the PCA figure. 208 209 height : int or float, default 6 210 Height of the PCA figure. 211 212 Returns 213 ------- 214 matplotlib.figure.Figure 215 Scatter plot showing the first two principal components. 216 217 Updates 218 ------- 219 self.pca : pandas.DataFrame 220 DataFrame with principal component scores for each sample. 221 222 self.explained_var : numpy.ndarray 223 Percentage of variance explained by each principal component. 224 225 self.cumulative_var : numpy.ndarray 226 Cumulative explained variance. 227 """ 228 229 scaler = StandardScaler() 230 data_scaled = scaler.fit_transform(self.clustering_data.T) 231 232 if pc_num == 0 or pc_num > data_scaled.shape[0]: 233 pc_num = data_scaled.shape[0] 234 235 pca = PCA(n_components=pc_num, random_state=42) 236 237 principal_components = pca.fit_transform(data_scaled) 238 239 pca_df = pd.DataFrame( 240 data=principal_components, 241 columns=["PC" + str(x + 1) for x in range(pc_num)], 242 ) 243 244 self.explained_var = pca.explained_variance_ratio_ * 100 245 self.cumulative_var = np.cumsum(self.explained_var) 246 247 self.pca = pca_df 248 249 fig = plt.figure(figsize=(width, height)) 250 plt.scatter(pca_df["PC1"], pca_df["PC2"], alpha=0.7) 251 plt.xlabel("PC 1") 252 plt.ylabel("PC 2") 253 plt.grid(True) 254 plt.show() 255 256 return fig 257 258 def knee_plot_PCA(self, width: int = 8, height: int = 6): 259 """ 260 Plot cumulative explained variance to determine the optimal number of PCs. 261 262 Parameters 263 ---------- 264 width : int, default 8 265 Width of the figure. 266 267 height : int or, default 6 268 Height of the figure. 269 270 Returns 271 ------- 272 matplotlib.figure.Figure 273 Line plot showing cumulative variance explained by each PC. 274 """ 275 276 fig_knee = plt.figure(figsize=(width, height)) 277 plt.plot(range(1, len(self.explained_var) + 1), self.cumulative_var, marker="o") 278 plt.xlabel("PC (n components)") 279 plt.ylabel("Cumulative explained variance (%)") 280 plt.grid(True) 281 282 xticks = [1] + list(range(5, len(self.explained_var) + 1, 5)) 283 plt.xticks(xticks, rotation=60) 284 285 plt.show() 286 287 return fig_knee 288 289 def find_clusters_PCA( 290 self, 291 pc_num: int = 2, 292 eps: float = 0.5, 293 min_samples: int = 10, 294 width: int = 8, 295 height: int = 6, 296 harmonized: bool = False, 297 ): 298 """ 299 Apply DBSCAN clustering to PCA embeddings and visualize the results. 300 301 This method performs density-based clustering (DBSCAN) on the PCA-reduced 302 dataset. Cluster labels are stored in the object's metadata, and a scatter 303 plot of the first two principal components with cluster annotations is returned. 304 305 Parameters 306 ---------- 307 pc_num : int, default 2 308 Number of principal components to use for clustering. 309 If 0, uses all available components. 310 311 eps : float, default 0.5 312 Maximum distance between two points for them to be considered 313 as neighbors (DBSCAN parameter). 314 315 min_samples : int, default 10 316 Minimum number of samples required to form a cluster (DBSCAN parameter). 317 318 width : int, default 8 319 Width of the output scatter plot. 320 321 height : int, default 6 322 Height of the output scatter plot. 323 324 harmonized : bool, default False 325 If True, use harmonized PCA data (`self.harmonized_pca`). 326 If False, use standard PCA results (`self.pca`). 327 328 Returns 329 ------- 330 matplotlib.figure.Figure 331 Scatter plot of the first two principal components colored by 332 cluster assignments. 333 334 Updates 335 ------- 336 self.clustering_metadata['PCA_clusters'] : list 337 Cluster labels assigned to each cell/sample. 338 339 self.input_metadata['PCA_clusters'] : list, optional 340 Cluster labels stored in input metadata (if available). 341 """ 342 343 dbscan = DBSCAN(eps=eps, min_samples=min_samples) 344 345 if pc_num == 0 and harmonized: 346 PCA = self.harmonized_pca 347 348 elif pc_num == 0: 349 PCA = self.pca 350 351 else: 352 if harmonized: 353 354 PCA = self.harmonized_pca.iloc[:, 0:pc_num] 355 else: 356 357 PCA = self.pca.iloc[:, 0:pc_num] 358 359 dbscan_labels = dbscan.fit_predict(PCA) 360 361 pca_df = pd.DataFrame(PCA) 362 pca_df["Cluster"] = dbscan_labels 363 364 fig = plt.figure(figsize=(width, height)) 365 366 for cluster_id in sorted(pca_df["Cluster"].unique()): 367 cluster_data = pca_df[pca_df["Cluster"] == cluster_id] 368 plt.scatter( 369 cluster_data["PC1"], 370 cluster_data["PC2"], 371 label=f"Cluster {cluster_id}", 372 alpha=0.7, 373 ) 374 375 plt.xlabel("PC 1") 376 plt.ylabel("PC 2") 377 plt.legend(title="Clusters", loc="center left", bbox_to_anchor=(1.0, 0.5)) 378 379 plt.grid(True) 380 plt.show() 381 382 self.clustering_metadata["PCA_clusters"] = [str(x) for x in dbscan_labels] 383 384 try: 385 self.input_metadata["PCA_clusters"] = [str(x) for x in dbscan_labels] 386 except: 387 pass 388 389 return fig 390 391 def perform_UMAP( 392 self, 393 factorize: bool = False, 394 umap_num: int = 100, 395 pc_num: int = 0, 396 harmonized: bool = False, 397 n_neighbors: int = 5, 398 min_dist: float | int = 0.1, 399 spread: float | int = 1.0, 400 set_op_mix_ratio: float | int = 1.0, 401 local_connectivity: int = 1, 402 repulsion_strength: float | int = 1.0, 403 negative_sample_rate: int = 5, 404 width: int = 8, 405 height: int = 6, 406 ): 407 """ 408 Compute and visualize UMAP embeddings of the dataset. 409 410 This method applies Uniform Manifold Approximation and Projection (UMAP) 411 for dimensionality reduction on either raw, PCA, or harmonized PCA data. 412 Results are stored as a DataFrame (`self.umap`) and a scatter plot figure 413 (`self.UMAP_plot`). 414 415 Parameters 416 ---------- 417 factorize : bool, default False 418 If True, categorical sample labels (from column names) are factorized 419 and used as supervision in UMAP fitting. 420 421 umap_num : int, default 100 422 Number of UMAP dimensions to compute. If 0, matches the input dimension. 423 424 pc_num : int, default 0 425 Number of principal components to use as UMAP input. 426 If 0, use all available components or raw data. 427 428 harmonized : bool, default False 429 If True, use harmonized PCA embeddings (`self.harmonized_pca`). 430 If False, use standard PCA or raw scaled data. 431 432 n_neighbors : int, default 5 433 UMAP parameter controlling the size of the local neighborhood. 434 435 min_dist : float, default 0.1 436 UMAP parameter controlling minimum allowed distance between embedded points. 437 438 spread : int | float, default 1.0 439 Effective scale of embedded space (UMAP parameter). 440 441 set_op_mix_ratio : int | float, default 1.0 442 Interpolation parameter between union and intersection in fuzzy sets. 443 444 local_connectivity : int, default 1 445 Number of nearest neighbors assumed for each point. 446 447 repulsion_strength : int | float, default 1.0 448 Weighting applied to negative samples during optimization. 449 450 negative_sample_rate : int, default 5 451 Number of negative samples per positive sample in optimization. 452 453 width : int, default 8 454 Width of the output scatter plot. 455 456 height : int, default 6 457 Height of the output scatter plot. 458 459 Updates 460 ------- 461 self.umap : pandas.DataFrame 462 Table of UMAP embeddings with columns `UMAP1 ... UMAPn`. 463 464 Notes 465 ----- 466 For supervised UMAP (`factorize=True`), categorical codes from column 467 names of the dataset are used as labels. 468 """ 469 470 scaler = StandardScaler() 471 472 if pc_num == 0 and harmonized: 473 data_scaled = self.harmonized_pca 474 475 elif pc_num == 0: 476 data_scaled = scaler.fit_transform(self.clustering_data.T) 477 478 else: 479 if harmonized: 480 481 data_scaled = self.harmonized_pca.iloc[:, 0:pc_num] 482 else: 483 484 data_scaled = self.pca.iloc[:, 0:pc_num] 485 486 if umap_num == 0 or umap_num > data_scaled.shape[1]: 487 488 umap_num = data_scaled.shape[1] 489 490 reducer = umap.UMAP( 491 n_components=len(data_scaled.T), 492 random_state=42, 493 n_neighbors=n_neighbors, 494 min_dist=min_dist, 495 spread=spread, 496 set_op_mix_ratio=set_op_mix_ratio, 497 local_connectivity=local_connectivity, 498 repulsion_strength=repulsion_strength, 499 negative_sample_rate=negative_sample_rate, 500 n_jobs=1, 501 ) 502 503 else: 504 505 reducer = umap.UMAP( 506 n_components=umap_num, 507 random_state=42, 508 n_neighbors=n_neighbors, 509 min_dist=min_dist, 510 spread=spread, 511 set_op_mix_ratio=set_op_mix_ratio, 512 local_connectivity=local_connectivity, 513 repulsion_strength=repulsion_strength, 514 negative_sample_rate=negative_sample_rate, 515 n_jobs=1, 516 ) 517 518 if factorize: 519 embedding = reducer.fit_transform( 520 X=data_scaled, y=pd.Categorical(self.clustering_data.columns).codes 521 ) 522 else: 523 embedding = reducer.fit_transform(X=data_scaled) 524 525 umap_df = pd.DataFrame( 526 embedding, columns=["UMAP" + str(x + 1) for x in range(umap_num)] 527 ) 528 529 plt.figure(figsize=(width, height)) 530 plt.scatter(umap_df["UMAP1"], umap_df["UMAP2"], alpha=0.7) 531 plt.xlabel("UMAP 1") 532 plt.ylabel("UMAP 2") 533 plt.grid(True) 534 535 plt.show() 536 537 self.umap = umap_df 538 539 def knee_plot_umap(self, eps: int | float = 0.5, min_samples: int = 10): 540 """ 541 Plot silhouette scores for different UMAP dimensions to determine optimal n_components. 542 543 Parameters 544 ---------- 545 eps : float, default 0.5 546 DBSCAN eps parameter for clustering each UMAP dimension. 547 548 min_samples : int, default 10 549 Minimum number of samples to form a cluster in DBSCAN. 550 551 Returns 552 ------- 553 matplotlib.figure.Figure 554 Silhouette score plot across UMAP dimensions. 555 """ 556 557 umap_range = range(2, len(self.umap.T) + 1) 558 559 silhouette_scores = [] 560 component = [] 561 for n in umap_range: 562 563 db = DBSCAN(eps=eps, min_samples=min_samples) 564 labels = db.fit_predict(np.array(self.umap)[:, :n]) 565 566 mask = labels != -1 567 if len(set(labels[mask])) > 1: 568 score = silhouette_score(np.array(self.umap)[:, :n][mask], labels[mask]) 569 else: 570 score = -1 571 572 silhouette_scores.append(score) 573 component.append(n) 574 575 fig = plt.figure(figsize=(10, 5)) 576 plt.plot(component, silhouette_scores, marker="o") 577 plt.xlabel("UMAP (n_components)") 578 plt.ylabel("Silhouette Score") 579 plt.grid(True) 580 plt.xticks(range(int(min(component)), int(max(component)) + 1, 1)) 581 582 plt.show() 583 584 return fig 585 586 def find_clusters_UMAP( 587 self, 588 umap_n: int = 5, 589 eps: float | float = 0.5, 590 min_samples: int = 10, 591 width: int = 8, 592 height: int = 6, 593 ): 594 """ 595 Apply DBSCAN clustering on UMAP embeddings and visualize clusters. 596 597 This method performs density-based clustering (DBSCAN) on the UMAP-reduced 598 dataset. Cluster labels are stored in the object's metadata, and a scatter 599 plot of the first two UMAP components with cluster annotations is returned. 600 601 Parameters 602 ---------- 603 umap_n : int, default 5 604 Number of UMAP dimensions to use for DBSCAN clustering. 605 Must be <= number of columns in `self.umap`. 606 607 eps : float | int, default 0.5 608 Maximum neighborhood distance between two samples for them to be considered 609 as in the same cluster (DBSCAN parameter). 610 611 min_samples : int, default 10 612 Minimum number of samples in a neighborhood to form a cluster (DBSCAN parameter). 613 614 width : int, default 8 615 Figure width. 616 617 height : int, default 6 618 Figure height. 619 620 Returns 621 ------- 622 matplotlib.figure.Figure 623 Scatter plot of the first two UMAP components colored by 624 cluster assignments. 625 626 Updates 627 ------- 628 self.clustering_metadata['UMAP_clusters'] : list 629 Cluster labels assigned to each cell/sample. 630 631 self.input_metadata['UMAP_clusters'] : list, optional 632 Cluster labels stored in input metadata (if available). 633 """ 634 635 dbscan = DBSCAN(eps=eps, min_samples=min_samples) 636 dbscan_labels = dbscan.fit_predict(np.array(self.umap)[:, :umap_n]) 637 638 umap_df = self.umap 639 umap_df["Cluster"] = dbscan_labels 640 641 fig = plt.figure(figsize=(width, height)) 642 643 for cluster_id in sorted(umap_df["Cluster"].unique()): 644 cluster_data = umap_df[umap_df["Cluster"] == cluster_id] 645 plt.scatter( 646 cluster_data["UMAP1"], 647 cluster_data["UMAP2"], 648 label=f"Cluster {cluster_id}", 649 alpha=0.7, 650 ) 651 652 plt.xlabel("UMAP 1") 653 plt.ylabel("UMAP 2") 654 plt.legend(title="Clusters", loc="center left", bbox_to_anchor=(1.0, 0.5)) 655 plt.grid(True) 656 657 self.clustering_metadata["UMAP_clusters"] = [str(x) for x in dbscan_labels] 658 659 try: 660 self.input_metadata["UMAP_clusters"] = [str(x) for x in dbscan_labels] 661 except: 662 pass 663 664 return fig 665 666 def UMAP_vis( 667 self, 668 names_slot: str = "cell_names", 669 set_sep: bool = True, 670 point_size: int | float = 0.6, 671 font_size: int | float = 6, 672 legend_split_col: int = 2, 673 width: int = 8, 674 height: int = 6, 675 inc_num: bool = True, 676 ): 677 """ 678 Visualize UMAP embeddings with sample labels based on specyfic metadata slot. 679 680 Parameters 681 ---------- 682 names_slot : str, default 'cell_names' 683 Column in metadata to use as sample labels. 684 685 set_sep : bool, default True 686 If True, separate points by dataset. 687 688 point_size : float, default 0.6 689 Size of scatter points. 690 691 font_size : int, default 6 692 Font size for numbers on points. 693 694 legend_split_col : int, default 2 695 Number of columns in legend. 696 697 width : int, default 8 698 Figure width. 699 700 height : int, default 6 701 Figure height. 702 703 inc_num : bool, default True 704 If True, annotate points with numeric labels. 705 706 Returns 707 ------- 708 matplotlib.figure.Figure 709 UMAP scatter plot figure. 710 """ 711 712 umap_df = self.umap.iloc[:, 0:2].copy() 713 umap_df["names"] = list(self.clustering_metadata[names_slot]) 714 715 if set_sep: 716 717 if "sets" in list(self.clustering_metadata.columns): 718 umap_df["dataset"] = list(self.clustering_metadata["sets"]) 719 else: 720 umap_df["dataset"] = "default" 721 722 else: 723 umap_df["dataset"] = "default" 724 725 umap_df["tmp_nam"] = list(umap_df["names"] + umap_df["dataset"]) 726 727 umap_df["count"] = umap_df["tmp_nam"].map(umap_df["tmp_nam"].value_counts()) 728 729 numeric_df = ( 730 pd.DataFrame(umap_df[["count", "tmp_nam", "names"]].copy()) 731 .drop_duplicates() 732 .sort_values("count", ascending=False) 733 ) 734 numeric_df["numeric_values"] = range(0, numeric_df.shape[0]) 735 736 umap_df = umap_df.merge( 737 numeric_df[["tmp_nam", "numeric_values"]], on="tmp_nam", how="left" 738 ) 739 740 fig, ax = plt.subplots(figsize=(width, height)) 741 742 markers = ["o", "s", "^", "D", "P", "*", "X"] 743 marker_map = { 744 ds: markers[i % len(markers)] 745 for i, ds in enumerate(umap_df["dataset"].unique()) 746 } 747 748 cord_list = [] 749 750 for num, nam in zip(numeric_df["numeric_values"], numeric_df["names"]): 751 752 cluster_data = umap_df[umap_df["numeric_values"] == num] 753 754 ax.scatter( 755 cluster_data["UMAP1"], 756 cluster_data["UMAP2"], 757 label=f"{num} - {nam}", 758 marker=marker_map[cluster_data["dataset"].iloc[0]], 759 alpha=0.6, 760 s=point_size, 761 ) 762 763 coords = cluster_data[["UMAP1", "UMAP2"]].values 764 765 dists = pairwise_distances(coords) 766 767 sum_dists = dists.sum(axis=1) 768 769 center_idx = np.argmin(sum_dists) 770 center_point = coords[center_idx] 771 772 cord_list.append(center_point) 773 774 if inc_num: 775 texts = [] 776 for (x, y), num in zip(cord_list, numeric_df["numeric_values"]): 777 texts.append( 778 ax.text( 779 x, 780 y, 781 str(num), 782 ha="center", 783 va="center", 784 fontsize=font_size, 785 color="black", 786 ) 787 ) 788 789 adjust_text(texts, arrowprops=dict(arrowstyle="-", color="gray", lw=0.5)) 790 791 ax.set_xlabel("UMAP 1") 792 ax.set_ylabel("UMAP 2") 793 794 ax.legend( 795 title="Clusters", 796 loc="center left", 797 bbox_to_anchor=(1.05, 0.5), 798 ncol=legend_split_col, 799 markerscale=5, 800 ) 801 802 ax.grid(True) 803 804 plt.tight_layout() 805 806 return fig 807 808 def UMAP_feature( 809 self, 810 feature_name: str, 811 features_data: pd.DataFrame | None, 812 point_size: int | float = 0.6, 813 font_size: int | float = 6, 814 width: int = 8, 815 height: int = 6, 816 palette="light", 817 ): 818 """ 819 Visualize UMAP embedding with expression levels of a selected feature. 820 821 Each point (cell) in the UMAP plot is colored according to the expression 822 value of the chosen feature, enabling interpretation of spatial patterns 823 of gene activity or metadata distribution in low-dimensional space. 824 825 Parameters 826 ---------- 827 feature_name : str 828 Name of the feature to plot. 829 830 features_data : pandas.DataFrame or None, default None 831 If None, the function uses the DataFrame containing the clustering data. 832 To plot features not used in clustering, provide a wider DataFrame 833 containing the original feature values. 834 835 point_size : float, default 0.6 836 Size of scatter points in the plot. 837 838 font_size : int, default 6 839 Font size for axis labels and annotations. 840 841 width : int, default 8 842 Width of the matplotlib figure. 843 844 height : int, default 6 845 Height of the matplotlib figure. 846 847 palette : str, default 'light' 848 Color palette for expression visualization. Options are: 849 - 'light' 850 - 'dark' 851 - 'green' 852 - 'gray' 853 854 Returns 855 ------- 856 matplotlib.figure.Figure 857 UMAP scatter plot colored by feature values. 858 """ 859 860 umap_df = self.umap.iloc[:, 0:2].copy() 861 862 if features_data is None: 863 864 features_data = self.clustering_data 865 866 if features_data.shape[1] != umap_df.shape[0]: 867 raise ValueError( 868 "Imputed 'features_data' shape does not match the number of UMAP cells" 869 ) 870 871 blist = [ 872 True if x.upper() == feature_name.upper() else False 873 for x in features_data.index 874 ] 875 876 if not any(blist): 877 raise ValueError("Imputed feature_name is not included in the data") 878 879 umap_df.loc[:, "feature"] = ( 880 features_data.loc[blist, :] 881 .apply(lambda row: row.tolist(), axis=1) 882 .values[0] 883 ) 884 885 umap_df = umap_df.sort_values("feature", ascending=True) 886 887 import matplotlib.colors as mcolors 888 889 if palette == "light": 890 palette = px.colors.sequential.Sunsetdark 891 892 elif palette == "dark": 893 palette = px.colors.sequential.thermal 894 895 elif palette == "green": 896 palette = px.colors.sequential.Aggrnyl 897 898 elif palette == "gray": 899 palette = px.colors.sequential.gray 900 palette = palette[::-1] 901 902 else: 903 raise ValueError( 904 'Palette not found. Use: "light", "dark", "gray", or "green"' 905 ) 906 907 converted = [] 908 for c in palette: 909 rgb_255 = px.colors.unlabel_rgb(c) 910 rgb_01 = tuple(v / 255.0 for v in rgb_255) 911 converted.append(rgb_01) 912 913 my_cmap = mcolors.ListedColormap(converted, name="custom") 914 915 fig, ax = plt.subplots(figsize=(width, height)) 916 sc = ax.scatter( 917 umap_df["UMAP1"], 918 umap_df["UMAP2"], 919 c=umap_df["feature"], 920 s=point_size, 921 cmap=my_cmap, 922 alpha=1.0, 923 edgecolors="black", 924 linewidths=0.1, 925 ) 926 927 cbar = plt.colorbar(sc, ax=ax) 928 cbar.set_label(f"{feature_name}") 929 930 ax.set_xlabel("UMAP 1") 931 ax.set_ylabel("UMAP 2") 932 933 ax.grid(True) 934 935 plt.tight_layout() 936 937 return fig 938 939 def get_umap_data(self): 940 """ 941 Retrieve UMAP embedding data with optional cluster labels. 942 943 Returns the UMAP coordinates stored in `self.umap`. If clustering 944 metadata is available (specifically `UMAP_clusters`), the corresponding 945 cluster assignments are appended as an additional column. 946 947 Returns 948 ------- 949 pandas.DataFrame 950 DataFrame containing UMAP coordinates (columns: 'UMAP1', 'UMAP2', ...). 951 If available, includes an extra column 'clusters' with cluster labels. 952 953 Notes 954 ----- 955 - UMAP embeddings must be computed beforehand (e.g., using `perform_UMAP`). 956 - Cluster labels are added only if present in `self.clustering_metadata`. 957 """ 958 959 umap_data = self.umap 960 961 try: 962 umap_data["clusters"] = self.clustering_metadata["UMAP_clusters"] 963 except: 964 pass 965 966 return umap_data 967 968 def get_pca_data(self): 969 """ 970 Retrieve PCA embedding data with optional cluster labels. 971 972 Returns the principal component scores stored in `self.pca`. If clustering 973 metadata is available (specifically `PCA_clusters`), the corresponding 974 cluster assignments are appended as an additional column. 975 976 Returns 977 ------- 978 pandas.DataFrame 979 DataFrame containing PCA coordinates (columns: 'PC1', 'PC2', ...). 980 If available, includes an extra column 'clusters' with cluster labels. 981 982 Notes 983 ----- 984 - PCA must be computed beforehand (e.g., using `perform_PCA`). 985 - Cluster labels are added only if present in `self.clustering_metadata`. 986 """ 987 988 pca_data = self.pca 989 990 try: 991 pca_data["clusters"] = self.clustering_metadata["PCA_clusters"] 992 except: 993 pass 994 995 return pca_data 996 997 def return_clusters(self, clusters="umap"): 998 """ 999 Retrieve cluster labels from UMAP or PCA clustering results. 1000 1001 Parameters 1002 ---------- 1003 clusters : str, default 'umap' 1004 Source of cluster labels to return. Must be one of: 1005 - 'umap': return cluster labels from UMAP embeddings. 1006 - 'pca' : return cluster labels from PCA embeddings. 1007 1008 Returns 1009 ------- 1010 list 1011 Cluster labels corresponding to the selected embedding method. 1012 1013 Raises 1014 ------ 1015 ValueError 1016 If `clusters` is not 'umap' or 'pca'. 1017 1018 Notes 1019 ----- 1020 Requires that clustering has already been performed 1021 (e.g., using `find_clusters_UMAP` or `find_clusters_PCA`). 1022 """ 1023 1024 if clusters.lower() == "umap": 1025 clusters_vector = self.clustering_metadata["UMAP_clusters"] 1026 elif clusters.lower() == "pca": 1027 clusters_vector = self.clustering_metadata["PCA_clusters"] 1028 else: 1029 raise ValueError("Parameter 'clusters' must be either 'umap' or 'pca'.") 1030 1031 return clusters_vector 1032 1033 1034class COMPsc(Clustering): 1035 """ 1036 A class `COMPsc` (Comparison of single-cell data) designed for the integration, 1037 analysis, and visualization of single-cell datasets. 1038 The class supports independent dataset integration, subclustering of existing clusters, 1039 marker detection, and multiple visualization strategies. 1040 1041 The COMPsc class provides methods for: 1042 1043 - Normalizing and filtering single-cell data 1044 - Loading and saving sparse 10x-style datasets 1045 - Computing differential expression and marker genes 1046 - Clustering and subclustering analysis 1047 - Visualizing similarity and spatial relationships 1048 - Aggregating data by cell and set annotations 1049 - Managing metadata and renaming labels 1050 - Plotting gene detection histograms and feature dot plots 1051 1052 Methods 1053 ------- 1054 project_dir(path_to_directory, project_list) 1055 Scans a directory to create a COMPsc instance mapping project names to their paths. 1056 1057 save_project(name, path=os.getcwd()) 1058 Saves the COMPsc object to a pickle file on disk. 1059 1060 load_project(path) 1061 Loads a previously saved COMPsc object from a pickle file. 1062 1063 reduce_cols(reg, inc_set=False) 1064 Removes columns from data tables where column names contain a specified name or partial substring. 1065 1066 reduce_rows(reg, inc_set=False) 1067 Removes rows from data tables where column names contain a specified feature (gene) name. 1068 1069 get_data(set_info=False) 1070 Returns normalized data with optional set annotations in column names. 1071 1072 get_metadata() 1073 Returns the stored input metadata. 1074 1075 get_partial_data(names=None, features=None, name_slot='cell_names') 1076 Return a subset of the data by sample names and/or features. 1077 1078 gene_calculation() 1079 Calculates and stores per-cell gene detection counts as a pandas Series. 1080 1081 gene_histograme(bins=100) 1082 Plots a histogram of genes detected per cell with an overlaid normal distribution. 1083 1084 gene_threshold(min_n=None, max_n=None) 1085 Filters cells based on minimum and/or maximum gene detection thresholds. 1086 1087 load_sparse_from_projects(normalized_data=False) 1088 Loads and concatenates sparse 10x-style datasets from project paths into count or normalized data. 1089 1090 rename_names(mapping, slot='cell_names') 1091 Renames entries in a specified metadata column using a provided mapping dictionary. 1092 1093 rename_subclusters(mapping) 1094 Renames subcluster labels using a provided mapping dictionary. 1095 1096 save_sparse(path_to_save=os.getcwd(), name_slot='cell_names', data_slot='normalized') 1097 Exports data as 10x-compatible sparse files (matrix.mtx, barcodes.tsv, genes.tsv). 1098 1099 normalize_data(normalize=True, normalize_factor=100000) 1100 Normalizes raw counts to counts-per-specified factor (e.g., CPM-like). 1101 1102 statistic(cells=None, sets=None, min_exp=0.01, min_pct=0.1, n_proc=10) 1103 Computes per-feature differential expression statistics (Mann-Whitney U) comparing target vs. rest groups. 1104 1105 calculate_difference_markers(min_exp=0, min_pct=0.25, n_proc=10, force=False) 1106 Computes and caches differential markers using the statistic method. 1107 1108 clustering_features(features_list=None, name_slot='cell_names', p_val=0.05, top_n=25, adj_mean=True, beta=0.4) 1109 Prepares clustering input by selecting marker features and optionally smoothing cell values. 1110 1111 average() 1112 Aggregates normalized data by averaging across (cell_name, set) pairs. 1113 1114 estimating_similarity(method='pearson', p_val=0.05, top_n=25) 1115 Computes pairwise correlation and Euclidean distance between aggregated samples. 1116 1117 cell_genes_distribution(cells_names, cells_sets=None, image_width=10, image_high=7) 1118 Compares the distributions of detected genes per cell across selected cell populations using overlapping histograms. 1119 1120 mm_cell_genes(cells_names, cells_sets=None, non_include_zeros=True, image_width=7, image_high=10) 1121 Calculates and visualizes the mean and median gene expression across selected cell populations as a grouped bar plot. 1122 1123 cell_dispersion(cells_names, cells_sets=None, image_width=7, image_high=10) 1124 Assesses within-cluster transcriptional dispersion by calculating distances between individual cells and their cluster centroids, visualized via violin plots. 1125 1126 similarity_plot(split_sets=True, set_info=True, cmap='seismic', width=12, height=10) 1127 Visualizes pairwise similarity as a dot plot with correlation as hue and scaled distance as point size. 1128 1129 spatial_similarity(set_info=True, bandwidth=1, n_neighbors=5, min_dist=0.1, legend_split=2, point_size=20, ...) 1130 Creates a UMAP-like visualization of similarity relationships with cluster hulls and nearest-neighbor arrows. 1131 1132 subcluster_prepare(features, cluster) 1133 Initializes a Clustering object for subcluster analysis on a selected parent cluster. 1134 1135 define_subclusters(umap_num=2, eps=0.5, min_samples=10, bandwidth=1, n_neighbors=5, min_dist=0.1, ...) 1136 Performs UMAP and DBSCAN clustering on prepared subcluster data and stores cluster labels. 1137 1138 subcluster_features_dot(colors='viridis', hclust='complete', img_width=3, img_high=5, label_size=6, ...) 1139 Visualizes averaged expression and occurrence of features for subclusters as a dot plot. 1140 1141 subcluster_DEG_dot(top_n=3, min_exp=0, min_pct=0.25, p_val=0.05, colors='viridis', ...) 1142 Plots top differential features for subclusters as a features-dot visualization. 1143 1144 accept_subclusters() 1145 Commits subcluster labels to main metadata by renaming cell names and clears subcluster data. 1146 1147 Raises 1148 ------ 1149 ValueError 1150 For invalid parameters, mismatched dimensions, or missing metadata. 1151 1152 """ 1153 1154 def __init__( 1155 self, 1156 objects=None, 1157 ): 1158 """ 1159 Initialize the COMPsc class for single-cell data integration and analysis. 1160 1161 Parameters 1162 ---------- 1163 objects : list or None, optional 1164 Optional list of data objects to initialize the instance with. 1165 1166 Attributes 1167 ---------- 1168 -objects : list or None 1169 -input_data : pandas.DataFrame or None 1170 -input_metadata : pandas.DataFrame or None 1171 -normalized_data : pandas.DataFrame or None 1172 -agg_metadata : pandas.DataFrame or None 1173 -agg_normalized_data : pandas.DataFrame or None 1174 -similarity : pandas.DataFrame or None 1175 -var_data : pandas.DataFrame or None 1176 -subclusters_ : instance of Clustering class or None 1177 -cells_calc : pandas.Series or None 1178 -gene_calc : pandas.Series or None 1179 -composition_data : pandas.DataFrame or None 1180 """ 1181 1182 self.objects = objects 1183 """ Stores the input data objects.""" 1184 1185 self.input_data = None 1186 """Raw input data for clustering or integration analysis.""" 1187 1188 self.input_metadata = None 1189 """Metadata associated with the input data.""" 1190 1191 self.normalized_data = None 1192 """Normalized version of the input data.""" 1193 1194 self.agg_metadata = None 1195 '''Aggregated metadata for all sets in object related to "agg_normalized_data"''' 1196 1197 self.agg_normalized_data = None 1198 """Aggregated and normalized data across multiple sets.""" 1199 1200 self.similarity = None 1201 """Similarity data between cells across all samples. and sets""" 1202 1203 self.var_data = None 1204 """DEG analysis results summarizing variance across all samples in the object.""" 1205 1206 self.subclusters_ = None 1207 """Placeholder for information about subclusters analysis; if computed.""" 1208 1209 self.cells_calc = None 1210 """Number of cells detected per sample (grouped by lineage, e.g., cluster or name), reflecting data composition.""" 1211 1212 self.gene_calc = None 1213 """Number of genes detected per sample (cell), reflecting the sequencing depth.""" 1214 1215 self.composition_data = None 1216 """Data describing composition of cells across clusters or sets.""" 1217 1218 @classmethod 1219 def project_dir(cls, path_to_directory, project_list): 1220 """ 1221 Scan a directory and build a COMPsc instance mapping provided project names 1222 to their paths. 1223 1224 Parameters 1225 ---------- 1226 path_to_directory : str 1227 Path containing project subfolders. 1228 1229 project_list : list[str] 1230 List of filenames (folder names) to include in the returned object map. 1231 1232 Returns 1233 ------- 1234 COMPsc 1235 New COMPsc instance with `objects` populated. 1236 1237 Raises 1238 ------ 1239 Exception 1240 A generic exception is caught and a message printed if scanning fails. 1241 1242 Notes 1243 ----- 1244 Function attempts to match entries in `project_list` to directory 1245 names and constructs a simplified object key from the folder name. 1246 """ 1247 try: 1248 objects = {} 1249 for filename in tqdm(os.listdir(path_to_directory)): 1250 for c in project_list: 1251 f = os.path.join(path_to_directory, filename) 1252 if c == filename and os.path.isdir(f): 1253 objects[str(c)] = f 1254 1255 return cls(objects) 1256 1257 except: 1258 print("Something went wrong. Check the function input data and try again!") 1259 1260 def save_project(self, name, path: str = os.getcwd()): 1261 """ 1262 Save the COMPsc object to disk using pickle. 1263 1264 Parameters 1265 ---------- 1266 name : str 1267 Base filename (without extension) to use when saving. 1268 1269 path : str, default os.getcwd() 1270 Directory in which to save the project file. 1271 1272 Returns 1273 ------- 1274 None 1275 1276 Side Effects 1277 ------------ 1278 - Writes a file `<path>/<name>.jpkl` containing the pickled object. 1279 - Prints a confirmation message with saved path. 1280 """ 1281 1282 full = os.path.join(path, f"{name}.jpkl") 1283 1284 with open(full, "wb") as f: 1285 pickle.dump(self, f) 1286 1287 print(f"Project saved as {full}") 1288 1289 @classmethod 1290 def load_project(cls, path): 1291 """ 1292 Load a previously saved COMPsc project from a pickle file. 1293 1294 Parameters 1295 ---------- 1296 path : str 1297 Full path to the pickled project file. 1298 1299 Returns 1300 ------- 1301 COMPsc 1302 The unpickled COMPsc object. 1303 1304 Raises 1305 ------ 1306 FileNotFoundError 1307 If the provided path does not exist. 1308 """ 1309 1310 if not os.path.exists(path): 1311 raise FileNotFoundError("File does not exist!") 1312 with open(path, "rb") as f: 1313 obj = pickle.load(f) 1314 return obj 1315 1316 def reduce_cols( 1317 self, 1318 reg: str | None = None, 1319 full: str | None = None, 1320 name_slot: str = "cell_names", 1321 inc_set: bool = False, 1322 ): 1323 """ 1324 Remove columns (cells) whose names contain a substring `reg` or 1325 full name `full` from available tables. 1326 1327 Parameters 1328 ---------- 1329 reg : str | None 1330 Substring to search for in column/cell names; matching columns will be removed. 1331 If not None, `full` must be None. 1332 1333 full : str | None 1334 Full name to search for in column/cell names; matching columns will be removed. 1335 If not None, `reg` must be None. 1336 1337 name_slot : str, default 'cell_names' 1338 Column in metadata to use as sample names. 1339 1340 inc_set : bool, default False 1341 If True, column names are interpreted as 'cell_name # set' when matching. 1342 1343 Update 1344 ------------ 1345 Mutates `self.input_data`, `self.normalized_data`, `self.input_metadata`, 1346 `self.agg_normalized_data`, and `self.agg_metadata` (if they exist), 1347 removing columns/rows that match `reg`. 1348 1349 Raises 1350 ------ 1351 Raises ValueError if nothing matches the reduction mask. 1352 """ 1353 1354 if reg is None and full is None: 1355 raise ValueError( 1356 "Both 'reg' and 'full' arguments not provided. Please provide at least one of them!" 1357 ) 1358 1359 if reg is not None and full is not None: 1360 raise ValueError( 1361 "Both 'reg' and 'full' arguments are provided. " 1362 "Please provide only one of them!\n" 1363 "'reg' is used when only part of the name must be detected.\n" 1364 "'full' is used if the full name must be detected." 1365 ) 1366 1367 if reg is not None: 1368 1369 if self.input_data is not None: 1370 1371 if inc_set: 1372 1373 self.input_data.columns = ( 1374 self.input_metadata[name_slot] 1375 + " # " 1376 + self.input_metadata["sets"] 1377 ) 1378 1379 else: 1380 1381 self.input_data.columns = self.input_metadata[name_slot] 1382 1383 mask = [reg.upper() not in x.upper() for x in self.input_data.columns] 1384 1385 if len([y for y in mask if y is False]) == 0: 1386 raise ValueError("Nothing found to reduce") 1387 1388 self.input_data = self.input_data.loc[:, mask] 1389 1390 if self.normalized_data is not None: 1391 1392 if inc_set: 1393 1394 self.normalized_data.columns = ( 1395 self.input_metadata[name_slot] 1396 + " # " 1397 + self.input_metadata["sets"] 1398 ) 1399 1400 else: 1401 1402 self.normalized_data.columns = self.input_metadata[name_slot] 1403 1404 mask = [ 1405 reg.upper() not in x.upper() for x in self.normalized_data.columns 1406 ] 1407 1408 if len([y for y in mask if y is False]) == 0: 1409 raise ValueError("Nothing found to reduce") 1410 1411 self.normalized_data = self.normalized_data.loc[:, mask] 1412 1413 if self.input_metadata is not None: 1414 1415 if inc_set: 1416 1417 self.input_metadata["drop"] = ( 1418 self.input_metadata[name_slot] 1419 + " # " 1420 + self.input_metadata["sets"] 1421 ) 1422 1423 else: 1424 1425 self.input_metadata["drop"] = self.input_metadata[name_slot] 1426 1427 mask = [ 1428 reg.upper() not in x.upper() for x in self.input_metadata["drop"] 1429 ] 1430 1431 if len([y for y in mask if y is False]) == 0: 1432 raise ValueError("Nothing found to reduce") 1433 1434 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 1435 drop=True 1436 ) 1437 1438 self.input_metadata = self.input_metadata.drop( 1439 columns=["drop"], errors="ignore" 1440 ) 1441 1442 if self.agg_normalized_data is not None: 1443 1444 if inc_set: 1445 1446 self.agg_normalized_data.columns = ( 1447 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1448 ) 1449 1450 else: 1451 1452 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1453 1454 mask = [ 1455 reg.upper() not in x.upper() 1456 for x in self.agg_normalized_data.columns 1457 ] 1458 1459 if len([y for y in mask if y is False]) == 0: 1460 raise ValueError("Nothing found to reduce") 1461 1462 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 1463 1464 if self.agg_metadata is not None: 1465 1466 if inc_set: 1467 1468 self.agg_metadata["drop"] = ( 1469 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1470 ) 1471 1472 else: 1473 1474 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1475 1476 mask = [reg.upper() not in x.upper() for x in self.agg_metadata["drop"]] 1477 1478 if len([y for y in mask if y is False]) == 0: 1479 raise ValueError("Nothing found to reduce") 1480 1481 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 1482 drop=True 1483 ) 1484 1485 self.agg_metadata = self.agg_metadata.drop( 1486 columns=["drop"], errors="ignore" 1487 ) 1488 1489 elif full is not None: 1490 1491 if self.input_data is not None: 1492 1493 if inc_set: 1494 1495 self.input_data.columns = ( 1496 self.input_metadata[name_slot] 1497 + " # " 1498 + self.input_metadata["sets"] 1499 ) 1500 1501 if "#" not in full: 1502 1503 self.input_data.columns = self.input_metadata[name_slot] 1504 1505 print( 1506 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1507 "Only the names will be compared, without considering the set information." 1508 ) 1509 1510 else: 1511 1512 self.input_data.columns = self.input_metadata[name_slot] 1513 1514 mask = [full.upper() != x.upper() for x in self.input_data.columns] 1515 1516 if len([y for y in mask if y is False]) == 0: 1517 raise ValueError("Nothing found to reduce") 1518 1519 self.input_data = self.input_data.loc[:, mask] 1520 1521 if self.normalized_data is not None: 1522 1523 if inc_set: 1524 1525 self.normalized_data.columns = ( 1526 self.input_metadata[name_slot] 1527 + " # " 1528 + self.input_metadata["sets"] 1529 ) 1530 1531 if "#" not in full: 1532 1533 self.normalized_data.columns = self.input_metadata[name_slot] 1534 1535 print( 1536 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1537 "Only the names will be compared, without considering the set information." 1538 ) 1539 1540 else: 1541 1542 self.normalized_data.columns = self.input_metadata[name_slot] 1543 1544 mask = [full.upper() != x.upper() for x in self.normalized_data.columns] 1545 1546 if len([y for y in mask if y is False]) == 0: 1547 raise ValueError("Nothing found to reduce") 1548 1549 self.normalized_data = self.normalized_data.loc[:, mask] 1550 1551 if self.input_metadata is not None: 1552 1553 if inc_set: 1554 1555 self.input_metadata["drop"] = ( 1556 self.input_metadata[name_slot] 1557 + " # " 1558 + self.input_metadata["sets"] 1559 ) 1560 1561 if "#" not in full: 1562 1563 self.input_metadata["drop"] = self.input_metadata[name_slot] 1564 1565 print( 1566 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1567 "Only the names will be compared, without considering the set information." 1568 ) 1569 1570 else: 1571 1572 self.input_metadata["drop"] = self.input_metadata[name_slot] 1573 1574 mask = [full.upper() != x.upper() for x in self.input_metadata["drop"]] 1575 1576 if len([y for y in mask if y is False]) == 0: 1577 raise ValueError("Nothing found to reduce") 1578 1579 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 1580 drop=True 1581 ) 1582 1583 self.input_metadata = self.input_metadata.drop( 1584 columns=["drop"], errors="ignore" 1585 ) 1586 1587 if self.agg_normalized_data is not None: 1588 1589 if inc_set: 1590 1591 self.agg_normalized_data.columns = ( 1592 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1593 ) 1594 1595 if "#" not in full: 1596 1597 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1598 1599 print( 1600 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1601 "Only the names will be compared, without considering the set information." 1602 ) 1603 else: 1604 1605 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1606 1607 mask = [ 1608 full.upper() != x.upper() for x in self.agg_normalized_data.columns 1609 ] 1610 1611 if len([y for y in mask if y is False]) == 0: 1612 raise ValueError("Nothing found to reduce") 1613 1614 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 1615 1616 if self.agg_metadata is not None: 1617 1618 if inc_set: 1619 1620 self.agg_metadata["drop"] = ( 1621 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1622 ) 1623 1624 if "#" not in full: 1625 1626 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1627 1628 print( 1629 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1630 "Only the names will be compared, without considering the set information." 1631 ) 1632 else: 1633 1634 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1635 1636 mask = [full.upper() != x.upper() for x in self.agg_metadata["drop"]] 1637 1638 if len([y for y in mask if y is False]) == 0: 1639 raise ValueError("Nothing found to reduce") 1640 1641 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 1642 drop=True 1643 ) 1644 1645 self.agg_metadata = self.agg_metadata.drop( 1646 columns=["drop"], errors="ignore" 1647 ) 1648 1649 self.gene_calculation() 1650 self.cells_calculation() 1651 1652 def reduce_rows(self, features_list: list): 1653 """ 1654 Remove rows (features) whose names are included in features_list. 1655 1656 Parameters 1657 ---------- 1658 features_list : list 1659 List of features to search for in index/gene names; matching entries will be removed. 1660 1661 Update 1662 ------------ 1663 Mutates `self.input_data`, `self.normalized_data`, `self.input_metadata`, 1664 `self.agg_normalized_data`, and `self.agg_metadata` (if they exist), 1665 removing columns/rows that match `reg`. 1666 1667 Raises 1668 ------ 1669 Prints a message listing features that are not found in the data. 1670 """ 1671 1672 if self.input_data is not None: 1673 1674 res = find_features(self.input_data, features=features_list) 1675 1676 res_list = [x.upper() for x in res["included"]] 1677 1678 mask = [x.upper() not in res_list for x in self.input_data.index] 1679 1680 if len([y for y in mask if y is False]) == 0: 1681 raise ValueError("Nothing found to reduce") 1682 1683 self.input_data = self.input_data.loc[mask, :] 1684 1685 if self.normalized_data is not None: 1686 1687 res = find_features(self.normalized_data, features=features_list) 1688 1689 res_list = [x.upper() for x in res["included"]] 1690 1691 mask = [x.upper() not in res_list for x in self.normalized_data.index] 1692 1693 if len([y for y in mask if y is False]) == 0: 1694 raise ValueError("Nothing found to reduce") 1695 1696 self.normalized_data = self.normalized_data.loc[mask, :] 1697 1698 if self.agg_normalized_data is not None: 1699 1700 res = find_features(self.agg_normalized_data, features=features_list) 1701 1702 res_list = [x.upper() for x in res["included"]] 1703 1704 mask = [x.upper() not in res_list for x in self.agg_normalized_data.index] 1705 1706 if len([y for y in mask if y is False]) == 0: 1707 raise ValueError("Nothing found to reduce") 1708 1709 self.agg_normalized_data = self.agg_normalized_data.loc[mask, :] 1710 1711 if len(res["not_included"]) > 0: 1712 print("\nFeatures not found:") 1713 for i in res["not_included"]: 1714 print(i) 1715 1716 self.gene_calculation() 1717 self.cells_calculation() 1718 1719 def get_data(self, set_info: bool = False): 1720 """ 1721 Return normalized data with optional set annotation appended to column names. 1722 1723 Parameters 1724 ---------- 1725 set_info : bool, default False 1726 If True, column names are returned as "cell_name # set"; otherwise 1727 only the `cell_name` is used. 1728 1729 Returns 1730 ------- 1731 pandas.DataFrame 1732 The `self.normalized_data` table with columns renamed according to `set_info`. 1733 1734 Raises 1735 ------ 1736 AttributeError 1737 If `self.normalized_data` or `self.input_metadata` is missing. 1738 """ 1739 1740 to_return = self.normalized_data 1741 1742 if set_info: 1743 to_return.columns = ( 1744 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 1745 ) 1746 else: 1747 to_return.columns = self.input_metadata["cell_names"] 1748 1749 return to_return 1750 1751 def get_partial_data( 1752 self, 1753 names: list | str | None = None, 1754 features: list | str | None = None, 1755 name_slot: str = "cell_names", 1756 inc_metadata: bool = False, 1757 ): 1758 """ 1759 Return a subset of the data filtered by sample names and/or feature names. 1760 1761 Parameters 1762 ---------- 1763 names : list, str, or None 1764 Names of samples to include. If None, all samples are considered. 1765 1766 features : list, str, or None 1767 Names of features to include. If None, all features are considered. 1768 1769 name_slot : str 1770 Column in metadata to use as sample names. 1771 1772 inc_metadata : bool 1773 If True return tuple (data, metadata) 1774 1775 Returns 1776 ------- 1777 pandas.DataFrame 1778 Subset of the normalized data based on the specified names and features. 1779 """ 1780 1781 data = self.normalized_data.copy() 1782 metadata = self.input_metadata 1783 1784 if name_slot in self.input_metadata.columns: 1785 data.columns = self.input_metadata[name_slot] 1786 else: 1787 raise ValueError("'name_slot' not occured in data!'") 1788 1789 if isinstance(features, str): 1790 features = [features] 1791 elif features is None: 1792 features = [] 1793 1794 if isinstance(names, str): 1795 names = [names] 1796 elif names is None: 1797 names = [] 1798 1799 features = [x.upper() for x in features] 1800 names = [x.upper() for x in names] 1801 1802 columns_names = [x.upper() for x in data.columns] 1803 features_names = [x.upper() for x in data.index] 1804 1805 columns_bool = [True if x in names else False for x in columns_names] 1806 features_bool = [True if x in features else False for x in features_names] 1807 1808 if True not in columns_bool and True not in features_bool: 1809 print("Missing 'names' and/or 'features'. Returning full dataset instead.") 1810 1811 if True in columns_bool: 1812 data = data.loc[:, columns_bool] 1813 metadata = metadata.loc[columns_bool, :] 1814 1815 if True in features_bool: 1816 data = data.loc[features_bool, :] 1817 1818 not_in_features = [y for y in features if y not in features_names] 1819 1820 if len(not_in_features) > 0: 1821 print("\nThe following features were not found in data:") 1822 print("\n".join(not_in_features)) 1823 1824 not_in_names = [y for y in names if y not in columns_names] 1825 1826 if len(not_in_names) > 0: 1827 print("\nThe following names were not found in data:") 1828 print("\n".join(not_in_names)) 1829 1830 if inc_metadata: 1831 return data, metadata 1832 else: 1833 return data 1834 1835 def get_metadata(self): 1836 """ 1837 Return the stored input metadata. 1838 1839 Returns 1840 ------- 1841 pandas.DataFrame 1842 `self.input_metadata` (may be None if not set). 1843 """ 1844 1845 to_return = self.input_metadata 1846 1847 return to_return 1848 1849 def gene_calculation(self): 1850 """ 1851 Calculate and store per-cell counts (e.g., number of detected genes). 1852 1853 The method computes a binary (presence/absence) per cell and sums across 1854 features to produce `self.gene_calc`. 1855 1856 Update 1857 ------ 1858 Sets `self.gene_calc` as a pandas.Series. 1859 1860 Side Effects 1861 ------------ 1862 Uses `self.input_data` when available, otherwise `self.normalized_data`. 1863 """ 1864 1865 if self.input_data is not None: 1866 1867 bin_col = self.input_data.columns.copy() 1868 1869 bin_col = bin_col.where(bin_col <= 0, 1) 1870 1871 sum_data = bin_col.sum(axis=0) 1872 1873 self.gene_calc = sum_data 1874 1875 elif self.normalized_data is not None: 1876 1877 bin_col = self.normalized_data.copy() 1878 1879 bin_col = bin_col.where(bin_col <= 0, 1) 1880 1881 sum_data = bin_col.sum(axis=0) 1882 1883 self.gene_calc = sum_data 1884 1885 def gene_histograme(self, bins=100): 1886 """ 1887 Plot a histogram of the number of genes detected per cell. 1888 1889 Parameters 1890 ---------- 1891 bins : int, default 100 1892 Number of histogram bins. 1893 1894 Returns 1895 ------- 1896 matplotlib.figure.Figure 1897 Figure containing the histogram of gene contents. 1898 1899 Notes 1900 ----- 1901 Requires `self.gene_calc` to be computed prior to calling. 1902 """ 1903 1904 fig, ax = plt.subplots(figsize=(8, 5)) 1905 1906 _, bin_edges, _ = ax.hist( 1907 self.gene_calc, bins=bins, edgecolor="black", alpha=0.6 1908 ) 1909 1910 mu, sigma = np.mean(self.gene_calc), np.std(self.gene_calc) 1911 1912 x = np.linspace(min(self.gene_calc), max(self.gene_calc), 1000) 1913 y = norm.pdf(x, mu, sigma) 1914 1915 y_scaled = y * len(self.gene_calc) * (bin_edges[1] - bin_edges[0]) 1916 1917 ax.plot( 1918 x, y_scaled, "r-", linewidth=2, label=f"Normal(μ={mu:.2f}, σ={sigma:.2f})" 1919 ) 1920 1921 ax.set_xlabel("Value") 1922 ax.set_ylabel("Count") 1923 ax.set_title("Histogram of genes detected per cell") 1924 1925 ax.set_xticks(np.linspace(min(self.gene_calc), max(self.gene_calc), 20)) 1926 ax.tick_params(axis="x", rotation=90) 1927 1928 ax.legend() 1929 1930 return fig 1931 1932 def gene_threshold(self, min_n: int | None, max_n: int | None): 1933 """ 1934 Filter cells by gene-detection thresholds (min and/or max). 1935 1936 Parameters 1937 ---------- 1938 min_n : int or None 1939 Minimum number of detected genes required to keep a cell. 1940 1941 max_n : int or None 1942 Maximum number of detected genes allowed to keep a cell. 1943 1944 Update 1945 ------- 1946 Filters `self.input_data`, `self.normalized_data`, `self.input_metadata` 1947 (and calls `average()` if `self.agg_normalized_data` exists). 1948 1949 Side Effects 1950 ------------ 1951 Raises ValueError if both bounds are None or if filtering removes all cells. 1952 """ 1953 1954 if min_n is not None and max_n is not None: 1955 mask = (self.gene_calc > min_n) & (self.gene_calc < max_n) 1956 elif min_n is None and max_n is not None: 1957 mask = self.gene_calc < max_n 1958 elif min_n is not None and max_n is None: 1959 mask = self.gene_calc > min_n 1960 else: 1961 raise ValueError("Lack of both min_n and max_n values") 1962 1963 if self.input_data is not None: 1964 1965 if len([y for y in mask if y is False]) == 0: 1966 raise ValueError("Nothing to reduce") 1967 1968 self.input_data = self.input_data.loc[:, mask.values] 1969 1970 if self.normalized_data is not None: 1971 1972 if len([y for y in mask if y is False]) == 0: 1973 raise ValueError("Nothing to reduce") 1974 1975 self.normalized_data = self.normalized_data.loc[:, mask.values] 1976 1977 if self.input_metadata is not None: 1978 1979 if len([y for y in mask if y is False]) == 0: 1980 raise ValueError("Nothing to reduce") 1981 1982 self.input_metadata = self.input_metadata.loc[mask.values, :].reset_index( 1983 drop=True 1984 ) 1985 1986 self.input_metadata = self.input_metadata.drop( 1987 columns=["drop"], errors="ignore" 1988 ) 1989 1990 if self.agg_normalized_data is not None: 1991 self.average() 1992 1993 self.gene_calculation() 1994 self.cells_calculation() 1995 1996 def cells_calculation(self, name_slot="cell_names"): 1997 """ 1998 Calculate number of cells per call name / cluster. 1999 2000 The method computes a binary (presence/absence) per cell name / cluster and sums across 2001 cells. 2002 2003 Parameters 2004 ---------- 2005 name_slot : str, default 'cell_names' 2006 Column in metadata to use as sample names. 2007 2008 Update 2009 ------ 2010 Sets `self.cells_calc` as a pd.DataFrame. 2011 """ 2012 2013 ls = list(self.input_metadata[name_slot]) 2014 2015 df = pd.DataFrame( 2016 { 2017 "cluster": pd.Series(ls).value_counts().index, 2018 "n": pd.Series(ls).value_counts().values, 2019 } 2020 ) 2021 2022 self.cells_calc = df 2023 2024 def cell_histograme(self, name_slot: str = "cell_names"): 2025 """ 2026 Plot a histogram of the number of cells detected per cell name (cluster). 2027 2028 Parameters 2029 ---------- 2030 name_slot : str, default 'cell_names' 2031 Column in metadata to use as sample names. 2032 2033 Returns 2034 ------- 2035 matplotlib.figure.Figure 2036 Figure containing the histogram of cell contents. 2037 2038 Notes 2039 ----- 2040 Requires `self.cells_calc` to be computed prior to calling. 2041 """ 2042 2043 if name_slot != "cell_names": 2044 self.cells_calculation(name_slot=name_slot) 2045 2046 fig, ax = plt.subplots(figsize=(8, 5)) 2047 2048 _, bin_edges, _ = ax.hist( 2049 list(self.cells_calc["n"]), 2050 bins=len(set(self.cells_calc["cluster"])), 2051 edgecolor="black", 2052 color="orange", 2053 alpha=0.6, 2054 ) 2055 2056 mu, sigma = np.mean(list(self.cells_calc["n"])), np.std( 2057 list(self.cells_calc["n"]) 2058 ) 2059 2060 x = np.linspace( 2061 min(list(self.cells_calc["n"])), max(list(self.cells_calc["n"])), 1000 2062 ) 2063 y = norm.pdf(x, mu, sigma) 2064 2065 y_scaled = y * len(list(self.cells_calc["n"])) * (bin_edges[1] - bin_edges[0]) 2066 2067 ax.plot( 2068 x, y_scaled, "r-", linewidth=2, label=f"Normal(μ={mu:.2f}, σ={sigma:.2f})" 2069 ) 2070 2071 ax.set_xlabel("Value") 2072 ax.set_ylabel("Count") 2073 ax.set_title("Histogram of cells detected per cell name / cluster") 2074 2075 ax.set_xticks( 2076 np.linspace( 2077 min(list(self.cells_calc["n"])), max(list(self.cells_calc["n"])), 20 2078 ) 2079 ) 2080 ax.tick_params(axis="x", rotation=90) 2081 2082 ax.legend() 2083 2084 return fig 2085 2086 def cluster_threshold(self, min_n: int | None, name_slot: str = "cell_names"): 2087 """ 2088 Filter cell names / clusters by cell-detection threshold. 2089 2090 Parameters 2091 ---------- 2092 min_n : int or None 2093 Minimum number of detected genes required to keep a cell. 2094 2095 name_slot : str, default 'cell_names' 2096 Column in metadata to use as sample names. 2097 2098 2099 Update 2100 ------- 2101 Filters `self.input_data`, `self.normalized_data`, `self.input_metadata` 2102 (and calls `average()` if `self.agg_normalized_data` exists). 2103 """ 2104 2105 if name_slot != "cell_names": 2106 self.cells_calculation(name_slot=name_slot) 2107 2108 if min_n is not None: 2109 names = self.cells_calc["cluster"][self.cells_calc["n"] < min_n] 2110 else: 2111 raise ValueError("Lack of min_n value") 2112 2113 if len(names) > 0: 2114 2115 if self.input_data is not None: 2116 2117 self.input_data.columns = self.input_metadata[name_slot] 2118 2119 mask = [not any(r in x for r in names) for x in self.input_data.columns] 2120 2121 if len([y for y in mask if y is False]) > 0: 2122 2123 self.input_data = self.input_data.loc[:, mask] 2124 2125 if self.normalized_data is not None: 2126 2127 self.normalized_data.columns = self.input_metadata[name_slot] 2128 2129 mask = [ 2130 not any(r in x for r in names) for x in self.normalized_data.columns 2131 ] 2132 2133 if len([y for y in mask if y is False]) > 0: 2134 2135 self.normalized_data = self.normalized_data.loc[:, mask] 2136 2137 if self.input_metadata is not None: 2138 2139 self.input_metadata["drop"] = self.input_metadata[name_slot] 2140 2141 mask = [ 2142 not any(r in x for r in names) for x in self.input_metadata["drop"] 2143 ] 2144 2145 if len([y for y in mask if y is False]) > 0: 2146 2147 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 2148 drop=True 2149 ) 2150 2151 self.input_metadata = self.input_metadata.drop( 2152 columns=["drop"], errors="ignore" 2153 ) 2154 2155 if self.agg_normalized_data is not None: 2156 2157 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 2158 2159 mask = [ 2160 not any(r in x for r in names) 2161 for x in self.agg_normalized_data.columns 2162 ] 2163 2164 if len([y for y in mask if y is False]) > 0: 2165 2166 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 2167 2168 if self.agg_metadata is not None: 2169 2170 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 2171 2172 mask = [ 2173 not any(r in x for r in names) for x in self.agg_metadata["drop"] 2174 ] 2175 2176 if len([y for y in mask if y is False]) > 0: 2177 2178 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 2179 drop=True 2180 ) 2181 2182 self.agg_metadata = self.agg_metadata.drop( 2183 columns=["drop"], errors="ignore" 2184 ) 2185 2186 self.gene_calculation() 2187 self.cells_calculation() 2188 2189 def load_sparse_from_projects(self, normalized_data: bool = False): 2190 """ 2191 Load sparse 10x-style datasets from stored project paths, concatenate them, 2192 and populate `input_data` / `normalized_data` and `input_metadata`. 2193 2194 Parameters 2195 ---------- 2196 normalized_data : bool, default False 2197 If True, store concatenated tables in `self.normalized_data`. 2198 If False, store them in `self.input_data` and normalization 2199 is needed using normalize_data() method. 2200 2201 Side Effects 2202 ------------ 2203 - Reads each project using `load_sparse(...)` (expects matrix.mtx, genes.tsv, barcodes.tsv). 2204 - Concatenates all projects column-wise and sets `self.input_metadata`. 2205 - Replaces NaNs with zeros and updates `self.gene_calc`. 2206 """ 2207 2208 obj = self.objects 2209 2210 full_data = pd.DataFrame() 2211 full_metadata = pd.DataFrame() 2212 2213 for ke in obj.keys(): 2214 print(ke) 2215 2216 dt, met = load_sparse(path=obj[ke], name=ke) 2217 2218 full_data = pd.concat([full_data, dt], axis=1) 2219 full_metadata = pd.concat([full_metadata, met], axis=0) 2220 2221 full_data[np.isnan(full_data)] = 0 2222 2223 if normalized_data: 2224 self.normalized_data = full_data 2225 self.input_metadata = full_metadata 2226 else: 2227 2228 self.input_data = full_data 2229 self.input_metadata = full_metadata 2230 2231 self.gene_calculation() 2232 self.cells_calculation() 2233 2234 def rename_names(self, mapping: dict, slot: str = "cell_names"): 2235 """ 2236 Rename entries in `self.input_metadata[slot]` according to a provided mapping. 2237 2238 Parameters 2239 ---------- 2240 mapping : dict 2241 Dictionary with keys 'old_name' and 'new_name', each mapping to a list 2242 of equal length describing replacements. 2243 2244 slot : str, default 'cell_names' 2245 Metadata column to operate on. 2246 2247 Update 2248 ------- 2249 Updates `self.input_metadata[slot]` in-place with renamed values. 2250 2251 Raises 2252 ------ 2253 ValueError 2254 If mapping keys are incorrect, lengths differ, or some 'old_name' values 2255 are not present in the metadata column. 2256 """ 2257 2258 if set(["old_name", "new_name"]) != set(mapping.keys()): 2259 raise ValueError( 2260 "Mapping dictionary must contain keys 'old_name' and 'new_name', " 2261 "each with a list of names to change." 2262 ) 2263 2264 if len(mapping["old_name"]) != len(mapping["new_name"]): 2265 raise ValueError( 2266 "Mapping dictionary lists 'old_name' and 'new_name' " 2267 "must have the same length!" 2268 ) 2269 2270 names_vector = list(self.input_metadata[slot]) 2271 2272 if not all(elem in names_vector for elem in list(mapping["old_name"])): 2273 raise ValueError( 2274 f"Some entries from 'old_name' do not exist in the names of slot {slot}." 2275 ) 2276 2277 replace_dict = dict(zip(mapping["old_name"], mapping["new_name"])) 2278 2279 names_vector_ret = [replace_dict.get(item, item) for item in names_vector] 2280 2281 self.input_metadata[slot] = names_vector_ret 2282 2283 def rename_subclusters(self, mapping): 2284 """ 2285 Rename labels stored in `self.subclusters_.subclusters` according to mapping. 2286 2287 Parameters 2288 ---------- 2289 mapping : dict 2290 Mapping with keys 'old_name' and 'new_name' (lists of equal length). 2291 2292 Update 2293 ------- 2294 Updates `self.subclusters_.subclusters` with renamed labels. 2295 2296 Raises 2297 ------ 2298 ValueError 2299 If mapping is invalid or old names are not present. 2300 """ 2301 2302 if set(["old_name", "new_name"]) != set(mapping.keys()): 2303 raise ValueError( 2304 "Mapping dictionary must contain keys 'old_name' and 'new_name', " 2305 "each with a list of names to change." 2306 ) 2307 2308 if len(mapping["old_name"]) != len(mapping["new_name"]): 2309 raise ValueError( 2310 "Mapping dictionary lists 'old_name' and 'new_name' " 2311 "must have the same length!" 2312 ) 2313 2314 names_vector = list(self.subclusters_.subclusters) 2315 2316 if not all(elem in names_vector for elem in list(mapping["old_name"])): 2317 raise ValueError( 2318 "Some entries from 'old_name' do not exist in the subcluster names." 2319 ) 2320 2321 replace_dict = dict(zip(mapping["old_name"], mapping["new_name"])) 2322 2323 names_vector_ret = [replace_dict.get(item, item) for item in names_vector] 2324 2325 self.subclusters_.subclusters = names_vector_ret 2326 2327 def save_sparse( 2328 self, 2329 path_to_save: str = os.getcwd(), 2330 name_slot: str = "cell_names", 2331 data_slot: str = "normalized", 2332 ): 2333 """ 2334 Export data as 10x-compatible sparse files (matrix.mtx, barcodes.tsv, genes.tsv). 2335 2336 Parameters 2337 ---------- 2338 path_to_save : str, default current working directory 2339 Directory where files will be written. 2340 2341 name_slot : str, default 'cell_names' 2342 Metadata column providing cell names for barcodes.tsv. 2343 2344 data_slot : str, default 'normalized' 2345 Either 'normalized' (uses self.normalized_data) or 'count' (uses self.input_data). 2346 2347 Raises 2348 ------ 2349 ValueError 2350 If `data_slot` is not 'normalized' or 'count'. 2351 """ 2352 2353 names = self.input_metadata[name_slot] 2354 2355 if data_slot.lower() == "normalized": 2356 2357 features = list(self.normalized_data.index) 2358 mtx = sparse.csr_matrix(self.normalized_data) 2359 2360 elif data_slot.lower() == "count": 2361 2362 features = list(self.input_data.index) 2363 mtx = sparse.csr_matrix(self.input_data) 2364 2365 else: 2366 raise ValueError("'data_slot' must be included in 'normalized' or 'count'") 2367 2368 os.makedirs(path_to_save, exist_ok=True) 2369 2370 mmwrite(os.path.join(path_to_save, "matrix.mtx"), mtx) 2371 2372 pd.Series(names).to_csv( 2373 os.path.join(path_to_save, "barcodes.tsv"), 2374 index=False, 2375 header=False, 2376 sep="\t", 2377 ) 2378 2379 pd.Series(features).to_csv( 2380 os.path.join(path_to_save, "genes.tsv"), index=False, header=False, sep="\t" 2381 ) 2382 2383 def normalize_counts( 2384 self, normalize_factor: int = 100000, log_transform: bool = True 2385 ): 2386 """ 2387 Normalize raw counts to counts-per-(normalize_factor) 2388 (e.g., CPM, TPM - depending on normalize_factor). 2389 2390 Parameters 2391 ---------- 2392 normalize_factor : int, default 100000 2393 Scaling factor used after dividing by column sums. 2394 2395 log_transform : bool, default True 2396 If True, apply log2(x+1) transformation to normalized values. 2397 2398 Update 2399 ------- 2400 Sets `self.normalized_data` to normalized values (fills NaNs with 0). 2401 2402 Raises 2403 ------ 2404 ValueError 2405 If `self.input_data` is missing (cannot normalize). 2406 """ 2407 if self.input_data is None: 2408 raise ValueError("Input data is missing, cannot normalize.") 2409 2410 sum_col = self.input_data.sum() 2411 self.normalized_data = self.input_data.div(sum_col).fillna(0) * normalize_factor 2412 2413 if log_transform: 2414 # log2(x + 1) to avoid -inf for zeros 2415 self.normalized_data = np.log2(self.normalized_data + 1) 2416 2417 def statistic( 2418 self, 2419 cells=None, 2420 sets=None, 2421 min_exp: float = 0.01, 2422 min_pct: float = 0.1, 2423 n_proc: int = 10, 2424 ): 2425 """ 2426 Compute per-feature statistics (Mann–Whitney U) comparing target vs rest. 2427 2428 This is a wrapper similar to `calc_DEG` tailored to use `self.normalized_data` 2429 and `self.input_metadata`. It returns per-feature statistics including p-values, 2430 adjusted p-values, means, variances, effect-size measures and fold-changes. 2431 2432 Parameters 2433 ---------- 2434 cells : list, 'All', dict, or None 2435 Defines the target cells or groups for comparison (several modes supported). 2436 2437 sets : 'All', dict, or None 2438 Alternative grouping mode (operate on `self.input_metadata['sets']`). 2439 2440 min_exp : float, default 0.01 2441 Minimum expression threshold used when filtering features. 2442 2443 min_pct : float, default 0.1 2444 Minimum proportion of expressing cells in the target group required to test a feature. 2445 2446 n_proc : int, default 10 2447 Number of parallel jobs to use. 2448 2449 Returns 2450 ------- 2451 pandas.DataFrame or dict 2452 Results DataFrame (or dict containing valid/control cells + DataFrame), 2453 similar to `calc_DEG` interface. 2454 2455 Raises 2456 ------ 2457 ValueError 2458 If neither `cells` nor `sets` is provided, or input metadata mismatch occurs. 2459 2460 Notes 2461 ----- 2462 Multiple modes supported: single-list entities, 'All', pairwise dicts, etc. 2463 """ 2464 2465 offset = 1e-100 2466 2467 def stat_calc(choose, feature_name): 2468 target_values = choose.loc[choose["DEG"] == "target", feature_name] 2469 rest_values = choose.loc[choose["DEG"] == "rest", feature_name] 2470 2471 pct_valid = (target_values > 0).sum() / len(target_values) 2472 pct_rest = (rest_values > 0).sum() / len(rest_values) 2473 2474 avg_valid = np.mean(target_values) 2475 avg_ctrl = np.mean(rest_values) 2476 sd_valid = np.std(target_values, ddof=1) 2477 sd_ctrl = np.std(rest_values, ddof=1) 2478 esm = (avg_valid - avg_ctrl) / np.sqrt(((sd_valid**2 + sd_ctrl**2) / 2)) 2479 2480 if np.sum(target_values) == np.sum(rest_values): 2481 p_val = 1.0 2482 else: 2483 _, p_val = stats.mannwhitneyu( 2484 target_values, rest_values, alternative="two-sided" 2485 ) 2486 2487 return { 2488 "feature": feature_name, 2489 "p_val": p_val, 2490 "pct_valid": pct_valid, 2491 "pct_ctrl": pct_rest, 2492 "avg_valid": avg_valid, 2493 "avg_ctrl": avg_ctrl, 2494 "sd_valid": sd_valid, 2495 "sd_ctrl": sd_ctrl, 2496 "esm": esm, 2497 } 2498 2499 def prepare_and_run_stat(choose, valid_group, min_exp, min_pct, n_proc): 2500 2501 def safe_min_half(series): 2502 filtered = series[(series > ((2**-1074) * 2)) & (series.notna())] 2503 return filtered.min() / 2 if not filtered.empty else 0 2504 2505 tmp_dat = choose[choose["DEG"] == "target"] 2506 tmp_dat = tmp_dat.drop("DEG", axis=1) 2507 2508 counts = (tmp_dat > min_exp).sum(axis=0) 2509 2510 total_count = tmp_dat.shape[0] 2511 2512 info = pd.DataFrame( 2513 {"feature": list(tmp_dat.columns), "pct": list(counts / total_count)} 2514 ) 2515 2516 del tmp_dat 2517 2518 drop_col = info["feature"][info["pct"] <= min_pct] 2519 2520 if len(drop_col) + 1 == len(choose.columns): 2521 drop_col = info["feature"][info["pct"] == 0] 2522 2523 del info 2524 2525 choose = choose.drop(list(drop_col), axis=1) 2526 2527 results = Parallel(n_jobs=n_proc)( 2528 delayed(stat_calc)(choose, feature) 2529 for feature in tqdm(choose.columns[choose.columns != "DEG"]) 2530 ) 2531 2532 df = pd.DataFrame(results) 2533 df = df[(df["avg_valid"] > 0) | (df["avg_ctrl"] > 0)] 2534 2535 df["valid_group"] = valid_group 2536 df.sort_values(by="p_val", inplace=True) 2537 2538 num_tests = len(df) 2539 df["adj_pval"] = np.minimum( 2540 1, (df["p_val"] * num_tests) / np.arange(1, num_tests + 1) 2541 ) 2542 2543 valid_factor = safe_min_half(df["avg_valid"]) 2544 ctrl_factor = safe_min_half(df["avg_ctrl"]) 2545 2546 cv_factor = min(valid_factor, ctrl_factor) 2547 2548 if cv_factor == 0: 2549 cv_factor = max(valid_factor, ctrl_factor) 2550 2551 if not np.isfinite(cv_factor) or cv_factor == 0: 2552 cv_factor += offset 2553 2554 valid = df["avg_valid"].where( 2555 df["avg_valid"] != 0, df["avg_valid"] + cv_factor 2556 ) 2557 ctrl = df["avg_ctrl"].where(df["avg_ctrl"] != 0, df["avg_ctrl"] + cv_factor) 2558 2559 df["FC"] = valid / ctrl 2560 2561 df["log(FC)"] = np.log2(df["FC"]) 2562 df["norm_diff"] = df["avg_valid"] - df["avg_ctrl"] 2563 2564 return df 2565 2566 choose = self.normalized_data.copy().T 2567 2568 final_results = [] 2569 2570 if isinstance(cells, list) and sets is None: 2571 print("\nAnalysis started...\nComparing selected cells to the whole set...") 2572 choose.index = ( 2573 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2574 ) 2575 2576 if "#" not in cells[0]: 2577 choose.index = self.input_metadata["cell_names"] 2578 2579 print( 2580 "Not include the set info (name # set) in the 'cells' list. " 2581 "Only the names will be compared, without considering the set information." 2582 ) 2583 2584 labels = ["target" if idx in cells else "rest" for idx in choose.index] 2585 valid = list( 2586 set(choose.index[[i for i, x in enumerate(labels) if x == "target"]]) 2587 ) 2588 2589 choose["DEG"] = labels 2590 choose = choose[choose["DEG"] != "drop"] 2591 2592 result_df = prepare_and_run_stat( 2593 choose.reset_index(drop=True), 2594 valid_group=valid, 2595 min_exp=min_exp, 2596 min_pct=min_pct, 2597 n_proc=n_proc, 2598 ) 2599 return {"valid_cells": valid, "control_cells": "rest", "DEG": result_df} 2600 2601 elif cells == "All" and sets is None: 2602 print("\nAnalysis started...\nComparing each type of cell to others...") 2603 choose.index = ( 2604 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2605 ) 2606 unique_labels = set(choose.index) 2607 2608 for label in tqdm(unique_labels): 2609 print(f"\nCalculating statistics for {label}") 2610 labels = ["target" if idx == label else "rest" for idx in choose.index] 2611 choose["DEG"] = labels 2612 choose = choose[choose["DEG"] != "drop"] 2613 result_df = prepare_and_run_stat( 2614 choose.copy(), 2615 valid_group=label, 2616 min_exp=min_exp, 2617 min_pct=min_pct, 2618 n_proc=n_proc, 2619 ) 2620 final_results.append(result_df) 2621 2622 return pd.concat(final_results, ignore_index=True) 2623 2624 elif cells is None and sets == "All": 2625 print("\nAnalysis started...\nComparing each set/group to others...") 2626 choose.index = self.input_metadata["sets"] 2627 unique_sets = set(choose.index) 2628 2629 for label in tqdm(unique_sets): 2630 print(f"\nCalculating statistics for {label}") 2631 labels = ["target" if idx == label else "rest" for idx in choose.index] 2632 2633 choose["DEG"] = labels 2634 choose = choose[choose["DEG"] != "drop"] 2635 result_df = prepare_and_run_stat( 2636 choose.copy(), 2637 valid_group=label, 2638 min_exp=min_exp, 2639 min_pct=min_pct, 2640 n_proc=n_proc, 2641 ) 2642 final_results.append(result_df) 2643 2644 return pd.concat(final_results, ignore_index=True) 2645 2646 elif cells is None and isinstance(sets, dict): 2647 print("\nAnalysis started...\nComparing groups...") 2648 2649 choose.index = self.input_metadata["sets"] 2650 2651 group_list = list(sets.keys()) 2652 if len(group_list) != 2: 2653 print("Only pairwise group comparison is supported.") 2654 return None 2655 2656 labels = [ 2657 ( 2658 "target" 2659 if idx in sets[group_list[0]] 2660 else "rest" if idx in sets[group_list[1]] else "drop" 2661 ) 2662 for idx in choose.index 2663 ] 2664 choose["DEG"] = labels 2665 choose = choose[choose["DEG"] != "drop"] 2666 2667 result_df = prepare_and_run_stat( 2668 choose.reset_index(drop=True), 2669 valid_group=group_list[0], 2670 min_exp=min_exp, 2671 min_pct=min_pct, 2672 n_proc=n_proc, 2673 ) 2674 return result_df 2675 2676 elif isinstance(cells, dict) and sets is None: 2677 print("\nAnalysis started...\nComparing groups...") 2678 choose.index = ( 2679 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2680 ) 2681 2682 if "#" not in cells[list(cells.keys())[0]][0]: 2683 choose.index = self.input_metadata["cell_names"] 2684 2685 print( 2686 "Not include the set info (name # set) in the 'cells' dict. " 2687 "Only the names will be compared, without considering the set information." 2688 ) 2689 2690 group_list = list(cells.keys()) 2691 if len(group_list) != 2: 2692 print("Only pairwise group comparison is supported.") 2693 return None 2694 2695 labels = [ 2696 ( 2697 "target" 2698 if idx in cells[group_list[0]] 2699 else "rest" if idx in cells[group_list[1]] else "drop" 2700 ) 2701 for idx in choose.index 2702 ] 2703 2704 choose["DEG"] = labels 2705 choose = choose[choose["DEG"] != "drop"] 2706 2707 result_df = prepare_and_run_stat( 2708 choose.reset_index(drop=True), 2709 valid_group=group_list[0], 2710 min_exp=min_exp, 2711 min_pct=min_pct, 2712 n_proc=n_proc, 2713 ) 2714 2715 return result_df.reset_index(drop=True) 2716 2717 else: 2718 raise ValueError( 2719 "You must specify either 'cells' or 'sets' (or both). None were provided, which is not allowed for this analysis." 2720 ) 2721 2722 def calculate_difference_markers( 2723 self, min_exp=0, min_pct=0.25, n_proc=10, force: bool = False 2724 ): 2725 """ 2726 Compute differential markers (var_data) if not already present. 2727 2728 Parameters 2729 ---------- 2730 min_exp : float, default 0 2731 Minimum expression threshold passed to `statistic`. 2732 2733 min_pct : float, default 0.25 2734 Minimum percent expressed in target group. 2735 2736 n_proc : int, default 10 2737 Parallel jobs. 2738 2739 force : bool, default False 2740 If True, recompute even if `self.var_data` is present. 2741 2742 Update 2743 ------- 2744 Sets `self.var_data` to the result of `self.statistic(...)`. 2745 2746 Raise 2747 ------ 2748 ValueError if already computed and `force` is False. 2749 """ 2750 2751 if self.var_data is None or force: 2752 2753 self.var_data = self.statistic( 2754 cells="All", sets=None, min_exp=min_exp, min_pct=min_pct, n_proc=n_proc 2755 ) 2756 2757 else: 2758 raise ValueError( 2759 "self.calculate_difference_markers() has already been executed. " 2760 "The results are stored in self.var. " 2761 "If you want to recalculate with different statistics, please rerun the method with force=True." 2762 ) 2763 2764 def clustering_features( 2765 self, 2766 features_list: list | None, 2767 name_slot: str = "cell_names", 2768 p_val: float = 0.05, 2769 top_n: int = 25, 2770 adj_mean: bool = True, 2771 beta: float = 0.2, 2772 ): 2773 """ 2774 Prepare clustering input by selecting marker features and optionally smoothing cell values 2775 toward group means. 2776 2777 Parameters 2778 ---------- 2779 features_list : list or None 2780 If provided, use this list of features. If None, features are selected 2781 from `self.var_data` (adj_pval <= p_val, positive logFC) picking `top_n` per group. 2782 2783 name_slot : str, default 'cell_names' 2784 Metadata column used for naming. 2785 2786 p_val : float, default 0.05 2787 Adjusted p-value cutoff when selecting features automatically. 2788 2789 top_n : int, default 25 2790 Number of top features per valid group to keep if `features_list` is None. 2791 2792 adj_mean : bool, default True 2793 If True, adjust cell values toward group means using `beta`. 2794 2795 beta : float, default 0.2 2796 Adjustment strength toward group mean. 2797 2798 Update 2799 ------ 2800 Sets `self.clustering_data` and `self.clustering_metadata` to the selected subset, 2801 ready for PCA/UMAP/clustering. 2802 """ 2803 2804 if features_list is None or len(features_list) == 0: 2805 2806 if self.var_data is None: 2807 raise ValueError( 2808 "Lack of 'self.var_data'. Use self.calculate_difference_markers() method first." 2809 ) 2810 2811 df_tmp = self.var_data[self.var_data["adj_pval"] <= p_val] 2812 df_tmp = df_tmp[df_tmp["log(FC)"] > 0] 2813 df_tmp = ( 2814 df_tmp.sort_values( 2815 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 2816 ) 2817 .groupby("valid_group") 2818 .head(top_n) 2819 ) 2820 2821 feaures_list = list(set(df_tmp["feature"])) 2822 2823 data = self.get_partial_data( 2824 names=None, features=feaures_list, name_slot=name_slot 2825 ) 2826 data_avg = average(data) 2827 2828 if adj_mean: 2829 data = adjust_cells_to_group_mean(data=data, data_avg=data_avg, beta=beta) 2830 2831 self.clustering_data = data 2832 2833 self.clustering_metadata = self.input_metadata 2834 2835 def average(self): 2836 """ 2837 Aggregate normalized data by (cell_name, set) pairs computing the mean per group. 2838 2839 The method constructs new column names as "cell_name # set", averages columns 2840 sharing identical labels, and populates `self.agg_normalized_data` and `self.agg_metadata`. 2841 2842 Update 2843 ------ 2844 Sets `self.agg_normalized_data` (features x aggregated samples) and 2845 `self.agg_metadata` (DataFrame with 'cell_names' and 'sets'). 2846 """ 2847 2848 wide_data = self.normalized_data 2849 2850 wide_metadata = self.input_metadata 2851 2852 new_names = wide_metadata["cell_names"] + " # " + wide_metadata["sets"] 2853 2854 wide_data.columns = list(new_names) 2855 2856 aggregated_df = wide_data.T.groupby(level=0).mean().T 2857 2858 sets = [re.sub(".*# ", "", x) for x in aggregated_df.columns] 2859 names = [re.sub(" #.*", "", x) for x in aggregated_df.columns] 2860 2861 aggregated_df.columns = names 2862 aggregated_metadata = pd.DataFrame({"cell_names": names, "sets": sets}) 2863 2864 self.agg_metadata = aggregated_metadata 2865 self.agg_normalized_data = aggregated_df 2866 2867 def estimating_similarity( 2868 self, method="pearson", p_val: float = 0.05, top_n: int = 25 2869 ): 2870 """ 2871 Estimate pairwise similarity and Euclidean distance between aggregated samples. 2872 2873 Parameters 2874 ---------- 2875 method : str, default 'pearson' 2876 Correlation method to use (passed to pandas.DataFrame.corr()). 2877 2878 p_val : float, default 0.05 2879 Adjusted p-value cutoff used to select marker features from `self.var_data`. 2880 2881 top_n : int, default 25 2882 Number of top features per valid group to include. 2883 2884 Update 2885 ------- 2886 Computes a combined table with per-pair correlation and euclidean distance 2887 and stores it in `self.similarity`. 2888 """ 2889 2890 if self.var_data is None: 2891 raise ValueError( 2892 "Lack of 'self.var_data'. Use self.calculate_difference_markers() method first." 2893 ) 2894 2895 if self.agg_normalized_data is None: 2896 self.average() 2897 2898 metadata = self.agg_metadata 2899 data = self.agg_normalized_data 2900 2901 df_tmp = self.var_data[self.var_data["adj_pval"] <= p_val] 2902 df_tmp = df_tmp[df_tmp["log(FC)"] > 0] 2903 df_tmp = ( 2904 df_tmp.sort_values( 2905 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 2906 ) 2907 .groupby("valid_group") 2908 .head(top_n) 2909 ) 2910 2911 data = data.loc[list(set(df_tmp["feature"]))] 2912 2913 if len(set(metadata["sets"])) > 1: 2914 data.columns = data.columns + " # " + [x for x in metadata["sets"]] 2915 else: 2916 data = data.copy() 2917 2918 scaler = StandardScaler() 2919 2920 scaled_data = scaler.fit_transform(data) 2921 2922 scaled_df = pd.DataFrame(scaled_data, columns=data.columns) 2923 2924 cor = scaled_df.corr(method=method) 2925 cor_df = cor.stack().reset_index() 2926 cor_df.columns = ["cell1", "cell2", "correlation"] 2927 2928 distances = pdist(scaled_df.T, metric="euclidean") 2929 dist_mat = pd.DataFrame( 2930 squareform(distances), index=scaled_df.columns, columns=scaled_df.columns 2931 ) 2932 dist_df = dist_mat.stack().reset_index() 2933 dist_df.columns = ["cell1", "cell2", "euclidean_dist"] 2934 2935 full = pd.merge(cor_df, dist_df, on=["cell1", "cell2"]) 2936 2937 full = full[full["cell1"] != full["cell2"]] 2938 full = full.reset_index(drop=True) 2939 2940 self.similarity = full 2941 2942 def similarity_plot( 2943 self, 2944 split_sets=True, 2945 set_info: bool = True, 2946 cmap="seismic", 2947 width=12, 2948 height=10, 2949 ): 2950 """ 2951 Visualize similarity as a dot plot. 2952 2953 Parameters 2954 ---------- 2955 split_sets : bool, default True 2956 If True and set information is present, split plotting area roughly into two halves to visualize cross-set pairs. 2957 2958 set_info : bool, default True 2959 If True, keep the ' # set' annotation in labels; otherwise strip it. 2960 2961 cmap : str, default 'seismic' 2962 Color map for correlation (hue). 2963 2964 width : int, default 12 2965 Figure width. 2966 2967 height : int, default 10 2968 Figure height. 2969 2970 Returns 2971 ------- 2972 matplotlib.figure.Figure 2973 2974 Raises 2975 ------ 2976 ValueError 2977 If `self.similarity` is None. 2978 2979 Notes 2980 ----- 2981 The function filters pairs by z-scored euclidean distance > 0 to focus on closer pairs. 2982 """ 2983 2984 if self.similarity is None: 2985 raise ValueError( 2986 "Similarity data is missing. Please calculate similarity using self.estimating_similarity." 2987 ) 2988 2989 similarity_data = self.similarity 2990 2991 if " # " in similarity_data["cell1"][0]: 2992 similarity_data["set1"] = [ 2993 re.sub(".*# ", "", x) for x in similarity_data["cell1"] 2994 ] 2995 similarity_data["set2"] = [ 2996 re.sub(".*# ", "", x) for x in similarity_data["cell2"] 2997 ] 2998 2999 if split_sets and " # " in similarity_data["cell1"][0]: 3000 sets = list( 3001 set(list(similarity_data["set1"]) + list(similarity_data["set2"])) 3002 ) 3003 3004 mm = math.ceil(len(sets) / 2) 3005 3006 x_s = sets[0:mm] 3007 y_s = sets[mm : len(sets)] 3008 3009 similarity_data = similarity_data[similarity_data["set1"].isin(x_s)] 3010 similarity_data = similarity_data[similarity_data["set2"].isin(y_s)] 3011 3012 similarity_data = similarity_data.sort_values(["set1", "set2"]) 3013 3014 if set_info is False and " # " in similarity_data["cell1"][0]: 3015 similarity_data["cell1"] = [ 3016 re.sub(" #.*", "", x) for x in similarity_data["cell1"] 3017 ] 3018 similarity_data["cell2"] = [ 3019 re.sub(" #.*", "", x) for x in similarity_data["cell2"] 3020 ] 3021 3022 similarity_data["-euclidean_zscore"] = -zscore( 3023 similarity_data["euclidean_dist"] 3024 ) 3025 3026 similarity_data = similarity_data[similarity_data["-euclidean_zscore"] > 0] 3027 3028 fig = plt.figure(figsize=(width, height)) 3029 sns.scatterplot( 3030 data=similarity_data, 3031 x="cell1", 3032 y="cell2", 3033 hue="correlation", 3034 size="-euclidean_zscore", 3035 sizes=(1, 100), 3036 palette=cmap, 3037 alpha=1, 3038 edgecolor="black", 3039 ) 3040 3041 plt.xticks(rotation=90) 3042 plt.yticks(rotation=0) 3043 plt.xlabel("Cell 1") 3044 plt.ylabel("Cell 2") 3045 plt.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 3046 3047 plt.grid(True, alpha=0.6) 3048 3049 plt.tight_layout() 3050 3051 return fig 3052 3053 def spatial_similarity( 3054 self, 3055 set_info: bool = True, 3056 bandwidth=1, 3057 n_neighbors=5, 3058 min_dist=0.1, 3059 legend_split=2, 3060 point_size=100, 3061 spread=1.0, 3062 set_op_mix_ratio=1.0, 3063 local_connectivity=1, 3064 repulsion_strength=1.0, 3065 negative_sample_rate=5, 3066 threshold=0.1, 3067 width=12, 3068 height=10, 3069 ): 3070 """ 3071 Create a spatial UMAP-like visualization of similarity relationships between samples. 3072 3073 Parameters 3074 ---------- 3075 set_info : bool, default True 3076 If True, retain set information in labels. 3077 3078 bandwidth : float, default 1 3079 Bandwidth used by MeanShift for clustering polygons. 3080 3081 point_size : float, default 100 3082 Size of scatter points. 3083 3084 legend_split : int, default 2 3085 Number of columns in legend. 3086 3087 n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate : parameters passed to UMAP. 3088 3089 threshold : float, default 0.1 3090 Minimum text distance for label adjustment to avoid overlap. 3091 3092 width : int, default 12 3093 Figure width. 3094 3095 height : int, default 10 3096 Figure height. 3097 3098 Returns 3099 ------- 3100 matplotlib.figure.Figure 3101 3102 Raises 3103 ------ 3104 ValueError 3105 If `self.similarity` is None. 3106 3107 Notes 3108 ----- 3109 Builds a precomputed distance matrix combining correlation and euclidean distance, 3110 runs UMAP with metric='precomputed', then overlays cluster hulls (MeanShift + convex hull) 3111 and arrows to indicate nearest neighbors (minimal combined distance). 3112 """ 3113 3114 if self.similarity is None: 3115 raise ValueError( 3116 "Similarity data is missing. Please calculate similarity using self.estimating_similarity." 3117 ) 3118 3119 similarity_data = self.similarity 3120 3121 sim = similarity_data["correlation"] 3122 sim_scaled = (sim - sim.min()) / (sim.max() - sim.min()) 3123 eu_dist = similarity_data["euclidean_dist"] 3124 eu_dist_scaled = (eu_dist - eu_dist.min()) / (eu_dist.max() - eu_dist.min()) 3125 3126 similarity_data["combo_dist"] = (1 - sim_scaled) * eu_dist_scaled 3127 3128 # for nn target 3129 arrow_df = similarity_data.copy() 3130 arrow_df = similarity_data.loc[ 3131 similarity_data.groupby("cell1")["combo_dist"].idxmin() 3132 ] 3133 3134 cells = sorted(set(similarity_data["cell1"]) | set(similarity_data["cell2"])) 3135 combo_matrix = pd.DataFrame(0, index=cells, columns=cells, dtype=float) 3136 3137 for _, row in similarity_data.iterrows(): 3138 combo_matrix.loc[row["cell1"], row["cell2"]] = row["combo_dist"] 3139 combo_matrix.loc[row["cell2"], row["cell1"]] = row["combo_dist"] 3140 3141 umap_model = umap.UMAP( 3142 n_components=2, 3143 metric="precomputed", 3144 n_neighbors=n_neighbors, 3145 min_dist=min_dist, 3146 spread=spread, 3147 set_op_mix_ratio=set_op_mix_ratio, 3148 local_connectivity=local_connectivity, 3149 repulsion_strength=repulsion_strength, 3150 negative_sample_rate=negative_sample_rate, 3151 transform_seed=42, 3152 init="spectral", 3153 random_state=42, 3154 verbose=True, 3155 ) 3156 3157 coords = umap_model.fit_transform(combo_matrix.values) 3158 cell_names = list(combo_matrix.index) 3159 num_cells = len(cell_names) 3160 palette = sns.color_palette("tab20c", num_cells) 3161 3162 if "#" in cell_names[0]: 3163 avsets = set( 3164 [re.sub(".*# ", "", x) for x in similarity_data["cell1"]] 3165 + [re.sub(".*# ", "", x) for x in similarity_data["cell2"]] 3166 ) 3167 num_sets = len(avsets) 3168 color_indices = [i * len(palette) // num_sets for i in range(num_sets)] 3169 color_mapping_sets = { 3170 set_name: palette[i] for i, set_name in zip(color_indices, avsets) 3171 } 3172 color_mapping = { 3173 name: color_mapping_sets[re.sub(".*# ", "", name)] 3174 for i, name in enumerate(cell_names) 3175 } 3176 else: 3177 color_mapping = {name: palette[i] for i, name in enumerate(cell_names)} 3178 3179 meanshift = MeanShift(bandwidth=bandwidth) 3180 labels = meanshift.fit_predict(coords) 3181 3182 fig = plt.figure(figsize=(width, height)) 3183 ax = plt.gca() 3184 3185 unique_labels = set(labels) 3186 cluster_palette = sns.color_palette("hls", len(unique_labels)) 3187 3188 for label in unique_labels: 3189 if label == -1: 3190 continue 3191 cluster_coords = coords[labels == label] 3192 if len(cluster_coords) < 3: 3193 continue 3194 3195 hull = ConvexHull(cluster_coords) 3196 hull_points = cluster_coords[hull.vertices] 3197 3198 centroid = np.mean(hull_points, axis=0) 3199 expanded = hull_points + 0.05 * (hull_points - centroid) 3200 3201 poly = Polygon( 3202 expanded, 3203 closed=True, 3204 facecolor=cluster_palette[label], 3205 edgecolor="none", 3206 alpha=0.2, 3207 zorder=1, 3208 ) 3209 ax.add_patch(poly) 3210 3211 texts = [] 3212 for i, (x, y) in enumerate(coords): 3213 plt.scatter( 3214 x, 3215 y, 3216 s=point_size, 3217 color=color_mapping[cell_names[i]], 3218 edgecolors="black", 3219 linewidths=0.5, 3220 zorder=2, 3221 ) 3222 texts.append( 3223 ax.text( 3224 x, y, str(i), ha="center", va="center", fontsize=8, color="black" 3225 ) 3226 ) 3227 3228 def dist(p1, p2): 3229 return np.sqrt((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) 3230 3231 texts_to_adjust = [] 3232 for i, t1 in enumerate(texts): 3233 for j, t2 in enumerate(texts): 3234 if i >= j: 3235 continue 3236 d = dist( 3237 (t1.get_position()[0], t1.get_position()[1]), 3238 (t2.get_position()[0], t2.get_position()[1]), 3239 ) 3240 if d < threshold: 3241 if t1 not in texts_to_adjust: 3242 texts_to_adjust.append(t1) 3243 if t2 not in texts_to_adjust: 3244 texts_to_adjust.append(t2) 3245 3246 adjust_text( 3247 texts_to_adjust, 3248 expand_text=(1.0, 1.0), 3249 force_text=0.9, 3250 arrowprops=dict(arrowstyle="-", color="gray", lw=0.1), 3251 ax=ax, 3252 ) 3253 3254 for _, row in arrow_df.iterrows(): 3255 try: 3256 idx1 = cell_names.index(row["cell1"]) 3257 idx2 = cell_names.index(row["cell2"]) 3258 except ValueError: 3259 continue 3260 x1, y1 = coords[idx1] 3261 x2, y2 = coords[idx2] 3262 arrow = FancyArrowPatch( 3263 (x1, y1), 3264 (x2, y2), 3265 arrowstyle="->", 3266 color="gray", 3267 linewidth=1.5, 3268 alpha=0.5, 3269 mutation_scale=12, 3270 zorder=0, 3271 ) 3272 ax.add_patch(arrow) 3273 3274 if set_info is False and " # " in cell_names[0]: 3275 3276 legend_elements = [ 3277 Patch( 3278 facecolor=color_mapping[name], 3279 edgecolor="black", 3280 label=f"{i} – {re.sub(' #.*', '', name)}", 3281 ) 3282 for i, name in enumerate(cell_names) 3283 ] 3284 3285 else: 3286 3287 legend_elements = [ 3288 Patch( 3289 facecolor=color_mapping[name], 3290 edgecolor="black", 3291 label=f"{i} – {name}", 3292 ) 3293 for i, name in enumerate(cell_names) 3294 ] 3295 3296 plt.legend( 3297 handles=legend_elements, 3298 title="Cells", 3299 bbox_to_anchor=(1.05, 1), 3300 loc="upper left", 3301 ncol=legend_split, 3302 ) 3303 3304 plt.xlabel("UMAP 1") 3305 plt.ylabel("UMAP 2") 3306 plt.grid(False) 3307 plt.show() 3308 3309 return fig 3310 3311 # subclusters part 3312 3313 def subcluster_prepare(self, features: list, cluster: str): 3314 """ 3315 Prepare a `Clustering` object for subcluster analysis on a selected parent cluster. 3316 3317 Parameters 3318 ---------- 3319 features : list 3320 Features to include for subcluster analysis. 3321 3322 cluster : str 3323 Parent cluster name (used to select matching cells). 3324 3325 Update 3326 ------ 3327 Initializes `self.subclusters_` as a new `Clustering` instance containing the 3328 reduced data for the given cluster and stores `current_features` and `current_cluster`. 3329 """ 3330 3331 dat = self.normalized_data 3332 dat.columns = list(self.input_metadata["cell_names"]) 3333 3334 dat = reduce_data(self.normalized_data, features=features, names=[cluster]) 3335 3336 self.subclusters_ = Clustering(data=dat, metadata=None) 3337 3338 self.subclusters_.current_features = features 3339 self.subclusters_.current_cluster = cluster 3340 3341 def define_subclusters( 3342 self, 3343 umap_num: int = 2, 3344 eps: float = 0.5, 3345 min_samples: int = 10, 3346 n_neighbors: int = 5, 3347 min_dist: float = 0.1, 3348 spread: float = 1.0, 3349 set_op_mix_ratio: float = 1.0, 3350 local_connectivity: int = 1, 3351 repulsion_strength: float = 1.0, 3352 negative_sample_rate: int = 5, 3353 width=8, 3354 height=6, 3355 ): 3356 """ 3357 Compute UMAP and DBSCAN clustering within a previously prepared subcluster dataset. 3358 3359 Parameters 3360 ---------- 3361 umap_num : int, default 2 3362 Number of UMAP dimensions to compute. 3363 3364 eps : float, default 0.5 3365 DBSCAN eps parameter. 3366 3367 min_samples : int, default 10 3368 DBSCAN min_samples parameter. 3369 3370 n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate, width, height : 3371 Additional parameters passed to UMAP / plotting / MeanShift as appropriate. 3372 3373 Update 3374 ------- 3375 Stores cluster labels in `self.subclusters_.subclusters`. 3376 3377 Raises 3378 ------ 3379 RuntimeError 3380 If `self.subclusters_` has not been prepared. 3381 """ 3382 3383 if self.subclusters_ is None: 3384 raise RuntimeError( 3385 "Nothing to return. 'self.subcluster_prepare' was not conducted!" 3386 ) 3387 3388 self.subclusters_.perform_UMAP( 3389 factorize=False, 3390 umap_num=umap_num, 3391 pc_num=0, 3392 harmonized=False, 3393 n_neighbors=n_neighbors, 3394 min_dist=min_dist, 3395 spread=spread, 3396 set_op_mix_ratio=set_op_mix_ratio, 3397 local_connectivity=local_connectivity, 3398 repulsion_strength=repulsion_strength, 3399 negative_sample_rate=negative_sample_rate, 3400 width=width, 3401 height=height, 3402 ) 3403 3404 fig = self.subclusters_.find_clusters_UMAP( 3405 umap_n=umap_num, 3406 eps=eps, 3407 min_samples=min_samples, 3408 width=width, 3409 height=height, 3410 ) 3411 3412 clusters = self.subclusters_.return_clusters(clusters="umap") 3413 3414 self.subclusters_.subclusters = [str(x) for x in list(clusters)] 3415 3416 return fig 3417 3418 def subcluster_features_dot( 3419 self, 3420 colors="viridis", 3421 hclust="complete", 3422 scale=False, 3423 scale_axis: Literal["x", "y"] = "y", 3424 img_width=3, 3425 img_high=5, 3426 label_size=6, 3427 size_scale=70, 3428 y_lab="Genes", 3429 legend_lab="normalized", 3430 bbox_to_anchor_scale: int = 25, 3431 bbox_to_anchor_perc: tuple = (0.91, 0.63), 3432 ): 3433 """ 3434 Create a features-dot visualization for the subclusters (averaged and occurrence). 3435 3436 Parameters 3437 ---------- 3438 colors : str, default 'viridis' 3439 Colormap name passed to `features_dot`. 3440 3441 hclust : str or None 3442 Hierarchical clustering linkage to order rows/columns. 3443 3444 scale: bool, default False 3445 If True, expression data will be scaled (0–1) across the rows (features). 3446 3447 scale_axis : {'x', 'y'}, default 'y' 3448 The axis along which to scale the expression data to the [0, 1] range: 3449 - 'y': Scales across columns (per sample / cell). 3450 - 'x': Scales across rows (per feature / gene). 3451 Only effective if `scale=True`. 3452 3453 img_width, img_high : float 3454 Figure size. 3455 3456 label_size : int 3457 Font size for labels. 3458 3459 size_scale : int 3460 Bubble size scaling. 3461 3462 y_lab : str 3463 X axis label. 3464 3465 legend_lab : str 3466 Colorbar label. 3467 3468 bbox_to_anchor_scale : int, default=25 3469 Vertical scale (percentage) for positioning the colorbar. 3470 3471 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3472 Anchor position for the size legend (percent bubble legend). 3473 3474 Returns 3475 ------- 3476 matplotlib.figure.Figure 3477 3478 Raises 3479 ------ 3480 RuntimeError 3481 If subcluster preparation/definition has not been run. 3482 """ 3483 3484 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3485 raise RuntimeError( 3486 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3487 ) 3488 3489 dat = self.normalized_data 3490 dat.columns = list(self.input_metadata["cell_names"]) 3491 3492 dat = reduce_data( 3493 self.normalized_data, 3494 features=self.subclusters_.current_features, 3495 names=[self.subclusters_.current_cluster], 3496 ) 3497 3498 dat.columns = self.subclusters_.subclusters 3499 3500 avg = average(dat) 3501 occ = occurrence(dat) 3502 3503 scatter = features_dot( 3504 expression_data=avg, 3505 occurence_data=occ, 3506 features=None, 3507 scale=scale, 3508 scale_axis=scale_axis, 3509 metadata_list=None, 3510 colors=colors, 3511 hclust=hclust, 3512 img_width=img_width, 3513 img_high=img_high, 3514 label_size=label_size, 3515 size_scale=size_scale, 3516 y_lab=y_lab, 3517 legend_lab=legend_lab, 3518 bbox_to_anchor_scale=bbox_to_anchor_scale, 3519 bbox_to_anchor_perc=bbox_to_anchor_perc, 3520 ) 3521 3522 return scatter 3523 3524 def subcluster_DEG_dot( 3525 self, 3526 top_n=3, 3527 min_exp=0, 3528 min_pct=0.25, 3529 p_val=0.05, 3530 colors="viridis", 3531 hclust="complete", 3532 scale=False, 3533 scale_axis: Literal["x", "y"] = "y", 3534 img_width=3, 3535 img_high=5, 3536 label_size=6, 3537 size_scale=70, 3538 y_lab="Genes", 3539 legend_lab="normalized", 3540 bbox_to_anchor_scale: int = 25, 3541 bbox_to_anchor_perc: tuple = (0.91, 0.63), 3542 n_proc=10, 3543 ): 3544 """ 3545 Plot top differential features (DEGs) for subclusters as a features-dot. 3546 3547 Parameters 3548 ---------- 3549 top_n : int, default 3 3550 Number of top features per subcluster to show. 3551 3552 min_exp : float, default 0 3553 Minimum expression threshold passed to `statistic`. 3554 3555 min_pct : float, default 0.25 3556 Minimum percent expressed in target group. 3557 3558 p_val: float, default 0.05 3559 Maximum p-value for visualizing features. 3560 3561 n_proc : int, default 10 3562 Parallel jobs used for DEG calculation. 3563 3564 scale: bool, default False 3565 If True, expression_data will be scaled (0–1) across the rows (features). 3566 3567 scale_axis : {'x', 'y'}, default 'y' 3568 The axis along which to scale the expression data to the [0, 1] range: 3569 - 'y': Scales across columns (per sample / cell). 3570 - 'x': Scales across rows (per feature / gene). 3571 Only effective if `scale=True`. 3572 3573 colors : str, default='viridis' 3574 Colormap for expression values. 3575 3576 hclust : str or None, default='complete' 3577 Linkage method for hierarchical clustering. If None, no clustering 3578 is performed. 3579 3580 img_width : int or float, default=8 3581 Width of the plot in inches. 3582 3583 img_high : int or float, default=5 3584 Height of the plot in inches. 3585 3586 label_size : int, default=10 3587 Font size for axis labels and ticks. 3588 3589 size_scale : int or float, default=100 3590 Scaling factor for bubble sizes. 3591 3592 y_lab : str, default='Genes' 3593 Label for the x-axis. 3594 3595 legend_lab : str, default='normalized' 3596 Label for the colorbar legend. 3597 3598 bbox_to_anchor_scale : int, default=25 3599 Vertical scale (percentage) for positioning the colorbar. 3600 3601 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3602 Anchor position for the size legend (percent bubble legend). 3603 3604 Returns 3605 ------- 3606 matplotlib.figure.Figure 3607 3608 Raises 3609 ------ 3610 RuntimeError 3611 If subcluster preparation/definition has not been run. 3612 3613 Notes 3614 ----- 3615 Internally calls `calc_DEG` (or equivalent) to obtain statistics, filters 3616 by p-value and effect-size, selects top features per valid group and plots them. 3617 """ 3618 3619 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3620 raise RuntimeError( 3621 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3622 ) 3623 3624 dat = self.normalized_data 3625 dat.columns = list(self.input_metadata["cell_names"]) 3626 3627 dat = reduce_data( 3628 self.normalized_data, names=[self.subclusters_.current_cluster] 3629 ) 3630 3631 dat.columns = self.subclusters_.subclusters 3632 3633 deg_stats = calc_DEG( 3634 dat, 3635 metadata_list=None, 3636 entities="All", 3637 sets=None, 3638 min_exp=min_exp, 3639 min_pct=min_pct, 3640 n_proc=n_proc, 3641 ) 3642 3643 deg_stats = deg_stats[deg_stats["p_val"] <= p_val] 3644 deg_stats = deg_stats[deg_stats["log(FC)"] > 0] 3645 3646 deg_stats = ( 3647 deg_stats.sort_values( 3648 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 3649 ) 3650 .groupby("valid_group") 3651 .head(top_n) 3652 ) 3653 3654 dat = reduce_data(dat, features=list(set(deg_stats["feature"]))) 3655 3656 avg = average(dat) 3657 occ = occurrence(dat) 3658 3659 scatter = features_dot( 3660 expression_data=avg, 3661 occurence_data=occ, 3662 features=None, 3663 metadata_list=None, 3664 scale_axis=scale_axis, 3665 scale=scale, 3666 colors=colors, 3667 hclust=hclust, 3668 img_width=img_width, 3669 img_high=img_high, 3670 label_size=label_size, 3671 size_scale=size_scale, 3672 y_lab=y_lab, 3673 legend_lab=legend_lab, 3674 bbox_to_anchor_scale=bbox_to_anchor_scale, 3675 bbox_to_anchor_perc=bbox_to_anchor_perc, 3676 ) 3677 3678 return scatter 3679 3680 def accept_subclusters(self): 3681 """ 3682 Commit subcluster labels into the main `input_metadata` by renaming cell names. 3683 3684 The method replaces occurrences of the parent cluster name in `self.input_metadata['cell_names']` 3685 with the expanded names that include subcluster suffixes (via `add_subnames`), 3686 then clears `self.subclusters_`. 3687 3688 Update 3689 ------ 3690 Modifies `self.input_metadata['cell_names']`. 3691 3692 Resets `self.subclusters_` to None. 3693 3694 Raises 3695 ------ 3696 RuntimeError 3697 If `self.subclusters_` is not defined or subclusters were not computed. 3698 """ 3699 3700 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3701 raise RuntimeError( 3702 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3703 ) 3704 3705 new_meta = add_subnames( 3706 list(self.input_metadata["cell_names"]), 3707 parent_name=self.subclusters_.current_cluster, 3708 new_clusters=self.subclusters_.subclusters, 3709 ) 3710 3711 self.input_metadata["cell_names"] = new_meta 3712 3713 self.subclusters_ = None 3714 3715 def dot_plot( 3716 self, 3717 names: list | None = None, 3718 features: list | None = None, 3719 name_slot: str = "cell_names", 3720 scale=True, 3721 scale_axis: Literal["x", "y"] = "y", 3722 colors="viridis", 3723 hclust=None, 3724 img_width=15, 3725 img_high=1, 3726 label_size=10, 3727 size_scale=200, 3728 y_lab="Genes", 3729 legend_lab="log(CPM + 1)", 3730 set_box_size: float | int = 5, 3731 set_box_high: float | int = 5, 3732 bbox_to_anchor_scale=25, 3733 bbox_to_anchor_perc=(0.90, -0.24), 3734 bbox_to_anchor_group=(1.01, 0.4), 3735 ): 3736 """ 3737 Create a dot plot of selected features across samples inside project. 3738 3739 Each point represents a feature-sample pair, where the color encodes the 3740 expression value and the size encodes occurrence or relative abundance. 3741 Optionally, hierarchical clustering can be applied to order rows and columns. 3742 3743 Parameters 3744 ---------- 3745 names : list, str, or None 3746 Names of samples to include. If None, all samples are considered. 3747 3748 features : list, str, or None 3749 Names of features to include. If None, all features are considered. 3750 3751 name_slot : str 3752 Column in metadata to use as sample names. 3753 3754 scale: bool, default False 3755 If True, expression_data will be scaled (0–1) across the rows (features). 3756 3757 scale_axis : {'x', 'y'}, default 'y' 3758 The axis along which to scale the expression data to the [0, 1] range: 3759 - 'y': Scales across columns (per sample / cell). 3760 - 'x': Scales across rows (per feature / gene). 3761 Only effective if `scale=True`. 3762 3763 colors : str, default='viridis' 3764 Colormap for expression values. 3765 3766 hclust : str or None, default='complete' 3767 Linkage method for hierarchical clustering. If None, no clustering 3768 is performed. 3769 3770 img_width : int or float, default=8 3771 Width of the plot in inches. 3772 3773 img_high : int or float, default=5 3774 Height of the plot in inches. 3775 3776 label_size : int, default=10 3777 Font size for axis labels and ticks. 3778 3779 size_scale : int or float, default=100 3780 Scaling factor for bubble sizes. 3781 3782 y_lab : str, default='Genes' 3783 Label for the x-axis. 3784 3785 legend_lab : str, default='log(CPM + 1)' 3786 Label for the colorbar legend. 3787 3788 bbox_to_anchor_scale : int, default=25 3789 Vertical scale (percentage) for positioning the colorbar. 3790 3791 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3792 Anchor position for the size legend (percent bubble legend). 3793 3794 bbox_to_anchor_group : tuple, default=(1.01, 0.4) 3795 Anchor position for the group legend. 3796 3797 Returns 3798 ------- 3799 matplotlib.figure.Figure 3800 The generated dot plot figure. 3801 3802 Notes 3803 ----- 3804 Colors represent expression values normalized to the colormap. 3805 """ 3806 3807 prtd, met = self.get_partial_data( 3808 names=names, features=features, name_slot=name_slot, inc_metadata=True 3809 ) 3810 3811 prtd.columns = prtd.columns + "#" + met["sets"] 3812 3813 prtd_avg = average(prtd) 3814 3815 meta_sets = [re.sub(".*#", "", x) for x in prtd_avg.columns] 3816 3817 prtd_avg.columns = [re.sub("#.*", "", x) for x in prtd_avg.columns] 3818 3819 prtd_occ = occurrence(prtd) 3820 3821 prtd_occ.columns = [re.sub("#.*", "", x) for x in prtd_occ.columns] 3822 3823 fig_scatter = features_dot( 3824 expression_data=prtd_avg, 3825 occurence_data=prtd_occ, 3826 scale=scale, 3827 scale_axis=scale_axis, 3828 features=None, 3829 metadata_list=meta_sets, 3830 colors=colors, 3831 hclust=hclust, 3832 img_width=img_width, 3833 img_high=img_high, 3834 label_size=label_size, 3835 size_scale=size_scale, 3836 y_lab=y_lab, 3837 legend_lab=legend_lab, 3838 set_box_size=set_box_size, 3839 set_box_high=set_box_high, 3840 bbox_to_anchor_scale=bbox_to_anchor_scale, 3841 bbox_to_anchor_perc=bbox_to_anchor_perc, 3842 bbox_to_anchor_group=bbox_to_anchor_group, 3843 ) 3844 3845 return fig_scatter 3846 3847 def data_composition( 3848 self, 3849 features_count: list | None, 3850 name_slot: str = "cell_names", 3851 set_sep: bool = True, 3852 ): 3853 """ 3854 Compute composition of cell types in data set. 3855 3856 This function counts the occurrences of specific cells (e.g., cell types, subtypes) 3857 within metadata entries, calculates their relative percentages, and stores 3858 the results in `self.composition_data`. 3859 3860 Parameters 3861 ---------- 3862 features_count : list or None 3863 List of features (part or full names) to be counted. 3864 If None, all unique elements from the specified `name_slot` metadata field are used. 3865 3866 name_slot : str, default 'cell_names' 3867 Metadata field containing sample identifiers or labels. 3868 3869 set_sep : bool, default True 3870 If True and multiple sets are present in metadata, compute composition 3871 separately for each set. 3872 3873 Update 3874 ------- 3875 Stores results in `self.composition_data` as a pandas DataFrame with: 3876 - 'name': feature name 3877 - 'n': number of occurrences 3878 - 'pct': percentage of occurrences 3879 - 'set' (if applicable): dataset identifier 3880 """ 3881 3882 validated_list = list(self.input_metadata[name_slot]) 3883 sets = list(self.input_metadata["sets"]) 3884 3885 if features_count is None: 3886 features_count = list(set(self.input_metadata[name_slot])) 3887 3888 if set_sep and len(set(sets)) > 1: 3889 3890 final_res = pd.DataFrame() 3891 3892 for s in set(sets): 3893 print(s) 3894 3895 mask = [True if s == x else False for x in sets] 3896 3897 tmp_val_list = np.array(validated_list) 3898 3899 tmp_val_list = list(tmp_val_list[mask]) 3900 3901 res_dict = {"name": [], "n": [], "set": []} 3902 3903 for f in tqdm(features_count): 3904 res_dict["n"].append( 3905 sum(1 for element in tmp_val_list if f in element) 3906 ) 3907 res_dict["name"].append(f) 3908 res_dict["set"].append(s) 3909 res = pd.DataFrame(res_dict) 3910 res["pct"] = res["n"] / sum(res["n"]) * 100 3911 res["pct"] = res["pct"].round(2) 3912 3913 final_res = pd.concat([final_res, res]) 3914 3915 res = final_res.sort_values(["set", "pct"], ascending=[True, False]) 3916 3917 else: 3918 3919 res_dict = {"name": [], "n": []} 3920 3921 for f in tqdm(features_count): 3922 res_dict["n"].append( 3923 sum(1 for element in validated_list if f in element) 3924 ) 3925 res_dict["name"].append(f) 3926 3927 res = pd.DataFrame(res_dict) 3928 res["pct"] = res["n"] / sum(res["n"]) * 100 3929 res["pct"] = res["pct"].round(2) 3930 3931 res = res.sort_values("pct", ascending=False) 3932 3933 self.composition_data = res 3934 3935 def composition_pie( 3936 self, 3937 width=6, 3938 height=6, 3939 font_size=15, 3940 cmap: str = "tab20", 3941 legend_split_col: int = 1, 3942 offset_labels: float | int = 0.5, 3943 legend_bbox: tuple = (1.15, 0.95), 3944 ): 3945 """ 3946 Visualize the composition of cell lineages using pie charts. 3947 3948 Generates pie charts showing the relative proportions of features stored 3949 in `self.composition_data`. If multiple sets are present, a separate 3950 chart is drawn for each set. 3951 3952 Parameters 3953 ---------- 3954 width : int, default 6 3955 Width of the figure. 3956 3957 height : int, default 6 3958 Height of the figure (applied per set if multiple sets are plotted). 3959 3960 font_size : int, default 15 3961 Font size for labels and annotations. 3962 3963 cmap : str, default 'tab20' 3964 Colormap used for pie slices. 3965 3966 legend_split_col : int, default 1 3967 Number of columns in the legend. 3968 3969 offset_labels : float or int, default 0.5 3970 Spacing offset for label placement relative to pie slices. 3971 3972 legend_bbox : tuple, default (1.15, 0.95) 3973 Bounding box anchor position for the legend. 3974 3975 Returns 3976 ------- 3977 matplotlib.figure.Figure 3978 Pie chart visualization of composition data. 3979 """ 3980 3981 df = self.composition_data 3982 3983 if "set" in df.columns and len(set(df["set"])) > 1: 3984 3985 sets = list(set(df["set"])) 3986 fig, axes = plt.subplots(len(sets), 1, figsize=(width, height * len(sets))) 3987 3988 all_wedges = [] 3989 cmap = plt.get_cmap(cmap) 3990 3991 set_nam = len(set(df["name"])) 3992 3993 legend_labels = list(set(df["name"])) 3994 3995 colors = [cmap(i / set_nam) for i in range(set_nam)] 3996 3997 cmap_dict = dict(zip(legend_labels, colors)) 3998 3999 for idx, s in enumerate(sets): 4000 ax = axes[idx] 4001 tmp_df = df[df["set"] == s].reset_index(drop=True) 4002 4003 labels = [f"{row['pct']:.1f}%" for _, row in tmp_df.iterrows()] 4004 4005 wedges, _ = ax.pie( 4006 tmp_df["n"], 4007 startangle=90, 4008 labeldistance=1.05, 4009 colors=[cmap_dict[x] for x in tmp_df["name"]], 4010 wedgeprops={"linewidth": 0.5, "edgecolor": "black"}, 4011 ) 4012 4013 all_wedges.extend(wedges) 4014 4015 kw = dict(arrowprops=dict(arrowstyle="-"), zorder=0, va="center") 4016 n = 0 4017 for i, p in enumerate(wedges): 4018 ang = (p.theta2 - p.theta1) / 2.0 + p.theta1 4019 y = np.sin(np.deg2rad(ang)) 4020 x = np.cos(np.deg2rad(ang)) 4021 horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] 4022 connectionstyle = f"angle,angleA=0,angleB={ang}" 4023 kw["arrowprops"].update({"connectionstyle": connectionstyle}) 4024 if len(labels[i]) > 0: 4025 n += offset_labels 4026 ax.annotate( 4027 labels[i], 4028 xy=(x, y), 4029 xytext=(1.01 * x + (n * x / 4), 1.01 * y + (n * y / 4)), 4030 horizontalalignment=horizontalalignment, 4031 fontsize=font_size, 4032 weight="bold", 4033 **kw, 4034 ) 4035 4036 circle2 = plt.Circle((0, 0), 0.6, color="white", ec="black") 4037 ax.add_artist(circle2) 4038 4039 ax.text( 4040 0, 4041 0, 4042 f"{s}", 4043 ha="center", 4044 va="center", 4045 fontsize=font_size, 4046 weight="bold", 4047 ) 4048 4049 legend_handles = [ 4050 Patch(facecolor=cmap_dict[label], edgecolor="black", label=label) 4051 for label in legend_labels 4052 ] 4053 4054 fig.legend( 4055 handles=legend_handles, 4056 loc="center right", 4057 bbox_to_anchor=legend_bbox, 4058 ncol=legend_split_col, 4059 title="", 4060 ) 4061 4062 plt.tight_layout() 4063 plt.show() 4064 4065 else: 4066 4067 labels = [f"{row['pct']:.1f}%" for _, row in df.iterrows()] 4068 4069 legend_labels = [f"{row['name']}" for _, row in df.iterrows()] 4070 4071 cmap = plt.get_cmap(cmap) 4072 colors = [cmap(i / len(df)) for i in range(len(df))] 4073 4074 fig, ax = plt.subplots( 4075 figsize=(width, height), subplot_kw=dict(aspect="equal") 4076 ) 4077 4078 wedges, _ = ax.pie( 4079 df["n"], 4080 startangle=90, 4081 labeldistance=1.05, 4082 colors=colors, 4083 wedgeprops={"linewidth": 0.5, "edgecolor": "black"}, 4084 ) 4085 4086 kw = dict(arrowprops=dict(arrowstyle="-"), zorder=0, va="center") 4087 n = 0 4088 for i, p in enumerate(wedges): 4089 ang = (p.theta2 - p.theta1) / 2.0 + p.theta1 4090 y = np.sin(np.deg2rad(ang)) 4091 x = np.cos(np.deg2rad(ang)) 4092 horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] 4093 connectionstyle = "angle,angleA=0,angleB={}".format(ang) 4094 kw["arrowprops"].update({"connectionstyle": connectionstyle}) 4095 if len(labels[i]) > 0: 4096 n += offset_labels 4097 4098 ax.annotate( 4099 labels[i], 4100 xy=(x, y), 4101 xytext=(1.01 * x + (n * x / 4), y * 1.01 + (n * y / 4)), 4102 horizontalalignment=horizontalalignment, 4103 fontsize=font_size, 4104 weight="bold", 4105 **kw, 4106 ) 4107 4108 circle2 = plt.Circle((0, 0), 0.6, color="white") 4109 circle2.set_edgecolor("black") 4110 4111 p = plt.gcf() 4112 p.gca().add_artist(circle2) 4113 4114 ax.legend( 4115 wedges, 4116 legend_labels, 4117 title="", 4118 loc="center left", 4119 bbox_to_anchor=legend_bbox, 4120 ncol=legend_split_col, 4121 ) 4122 4123 plt.show() 4124 4125 return fig 4126 4127 def bar_composition( 4128 self, 4129 cmap="tab20b", 4130 width=2, 4131 height=6, 4132 font_size=15, 4133 legend_split_col: int = 1, 4134 legend_bbox: tuple = (1.3, 1), 4135 ): 4136 """ 4137 Visualize the composition of cell lineages using bar plots. 4138 4139 Produces bar plots showing the distribution of features stored in 4140 `self.composition_data`. If multiple sets are present, a separate 4141 bar is drawn for each set. Percentages are annotated alongside the bars. 4142 4143 Parameters 4144 ---------- 4145 cmap : str, default 'tab20b' 4146 Colormap used for stacked bars. 4147 4148 width : int, default 2 4149 Width of each subplot (per set). 4150 4151 height : int, default 6 4152 Height of the figure. 4153 4154 font_size : int, default 15 4155 Font size for labels and annotations. 4156 4157 legend_split_col : int, default 1 4158 Number of columns in the legend. 4159 4160 legend_bbox : tuple, default (1.3, 1) 4161 Bounding box anchor position for the legend. 4162 4163 Returns 4164 ------- 4165 matplotlib.figure.Figure 4166 Stacked bar plot visualization of composition data. 4167 """ 4168 4169 df = self.composition_data 4170 df["num"] = range(1, len(df) + 1) 4171 4172 if "set" in df.columns and len(set(df["set"])) > 1: 4173 4174 sets = list(set(df["set"])) 4175 fig, axes = plt.subplots(1, len(sets), figsize=(width * len(sets), height)) 4176 4177 cmap = plt.get_cmap(cmap) 4178 4179 set_nam = len(set(df["name"])) 4180 4181 legend_labels = list(set(df["name"])) 4182 4183 colors = [cmap(i / set_nam) for i in range(set_nam)] 4184 4185 cmap_dict = dict(zip(legend_labels, colors)) 4186 4187 for idx, s in enumerate(sets): 4188 ax = axes[idx] 4189 4190 tmp_df = df[df["set"] == s].reset_index(drop=True) 4191 4192 values = tmp_df["n"].values 4193 total = sum(values) 4194 values = [v / total * 100 for v in values] 4195 values = [round(v, 2) for v in values] 4196 4197 idx_max = np.argmax(values) 4198 correction = 100 - sum(values) 4199 values[idx_max] += correction 4200 4201 names = tmp_df["name"].values 4202 perc = tmp_df["pct"].values 4203 nums = tmp_df["num"].values 4204 4205 bottom = 0 4206 centers = [] 4207 for name, num, val, color in zip(names, nums, values, colors): 4208 ax.bar(s, val, bottom=bottom, color=cmap_dict[name], label=name) 4209 centers.append(bottom + val / 2) 4210 bottom += val 4211 4212 y_positions = np.linspace(centers[0], centers[-1], len(centers)) 4213 x_text = -0.8 4214 4215 for y_label, y_center, pct, num in zip( 4216 y_positions, centers, perc, nums 4217 ): 4218 ax.annotate( 4219 f"{pct:.1f}%", 4220 xy=(0, y_center), 4221 xycoords="data", 4222 xytext=(x_text, y_label), 4223 textcoords="data", 4224 ha="right", 4225 va="center", 4226 fontsize=font_size, 4227 arrowprops=dict( 4228 arrowstyle="->", 4229 lw=1, 4230 color="black", 4231 connectionstyle="angle3,angleA=0,angleB=90", 4232 ), 4233 ) 4234 4235 ax.set_ylim(0, 100) 4236 ax.set_xlabel(s, fontsize=font_size) 4237 ax.xaxis.label.set_rotation(30) 4238 4239 ax.set_xticks([]) 4240 ax.set_yticks([]) 4241 for spine in ax.spines.values(): 4242 spine.set_visible(False) 4243 4244 legend_handles = [ 4245 Patch(facecolor=cmap_dict[label], edgecolor="black", label=label) 4246 for label in legend_labels 4247 ] 4248 4249 fig.legend( 4250 handles=legend_handles, 4251 loc="center right", 4252 bbox_to_anchor=legend_bbox, 4253 ncol=legend_split_col, 4254 title="", 4255 ) 4256 4257 plt.tight_layout() 4258 plt.show() 4259 4260 else: 4261 4262 cmap = plt.get_cmap(cmap) 4263 4264 colors = [cmap(i / len(df)) for i in range(len(df))] 4265 4266 fig, ax = plt.subplots(figsize=(width, height)) 4267 4268 values = df["n"].values 4269 names = df["name"].values 4270 perc = df["pct"].values 4271 nums = df["num"].values 4272 4273 bottom = 0 4274 centers = [] 4275 for name, num, val, color in zip(names, nums, values, colors): 4276 ax.bar(0, val, bottom=bottom, color=color, label=f"{num}) {name}") 4277 centers.append(bottom + val / 2) 4278 bottom += val 4279 4280 y_positions = np.linspace(centers[0], centers[-1], len(centers)) 4281 x_text = -0.8 4282 4283 for y_label, y_center, pct, num in zip(y_positions, centers, perc, nums): 4284 ax.annotate( 4285 f"{num}) {pct}", 4286 xy=(0, y_center), 4287 xycoords="data", 4288 xytext=(x_text, y_label), 4289 textcoords="data", 4290 ha="right", 4291 va="center", 4292 fontsize=9, 4293 arrowprops=dict( 4294 arrowstyle="->", 4295 lw=1, 4296 color="black", 4297 connectionstyle="angle3,angleA=0,angleB=90", 4298 ), 4299 ) 4300 4301 ax.set_xticks([]) 4302 ax.set_yticks([]) 4303 for spine in ax.spines.values(): 4304 spine.set_visible(False) 4305 4306 ax.legend( 4307 title="Legend", 4308 bbox_to_anchor=legend_bbox, 4309 loc="upper left", 4310 ncol=legend_split_col, 4311 ) 4312 4313 plt.tight_layout() 4314 plt.show() 4315 4316 return fig 4317 4318 def cell_regression( 4319 self, 4320 cell_x: str, 4321 cell_y: str, 4322 set_x: str | None = None, 4323 set_y: str | None = None, 4324 min_log=0.5, 4325 n_max=20, 4326 min_exp=0.1, 4327 min_pct=0.5, 4328 n_proc=10, 4329 min_esm=0.5, 4330 p_val=0.05, 4331 adj=True, 4332 image_width=12, 4333 image_high=7, 4334 ): 4335 """ 4336 Perform linear regression between two selected cell populations and identify 4337 significant and top-ranked differentially expressed genes. 4338 4339 The function compares the expression profiles of two selected cells or cell 4340 populations using linear regression. Each gene is represented as a point in 4341 the regression plot, with expression in `cell_x` on the X-axis and expression 4342 in `cell_y` on the Y-axis. 4343 4344 Differential expression statistics are calculated using `calc_DEG`. Genes are 4345 first filtered according to the minimum absolute log fold change, minimum 4346 absolute effect size (ESM), and statistical significance. Among the 4347 significant genes, up to `n_max` genes are selected within each `valid_group`, 4348 prioritizing genes with the largest standardized absolute regression residuals 4349 and absolute ESM values. 4350 4351 The regression plot uses three point categories: 4352 - gray: genes that are not statistically significant, 4353 - green: statistically significant genes that pass the DEG thresholds, 4354 - blue: top `n_max` genes selected within each `valid_group`. 4355 4356 The selected top genes are additionally annotated with their gene names. 4357 The plot also displays the fitted linear regression line, R-squared, 4358 p-value, and regression equation. 4359 4360 Parameters 4361 ---------- 4362 cell_x : str 4363 Name of the first cell or cell population used as the X-axis. 4364 4365 cell_y : str 4366 Name of the second cell or cell population used as the Y-axis. 4367 4368 set_x : str or None 4369 Dataset or experimental set identifier corresponding to `cell_x`. 4370 Required when multiple datasets contain the same cell name. If None, 4371 the cell is selected only by its name. 4372 4373 set_y : str or None 4374 Dataset or experimental set identifier corresponding to `cell_y`. 4375 Required when multiple datasets contain the same cell name. If None, 4376 the cell is selected only by its name. 4377 4378 min_log : float, default=0.5 4379 Minimum absolute log fold change required for a gene to be considered 4380 for significant DEG selection. 4381 4382 n_max : int, default=20 4383 Maximum number of top genes selected from each `valid_group`. 4384 Genes are ranked primarily by standardized absolute regression residual 4385 and secondarily by absolute Effect Size - Cohen's d. 4386 4387 min_exp : float, default=0.1 4388 Minimum expression threshold passed to `calc_DEG`. 4389 4390 min_pct : float, default=0.5 4391 Minimum fraction of cells in which a gene must be expressed, passed to 4392 `calc_DEG`. 4393 4394 n_proc : int, default=10 4395 Number of parallel processes used by `calc_DEG`. 4396 4397 min_esm : float, default=0.5 4398 Minimum absolute ESM (effect size measure) required for DEG selection. 4399 4400 p_val : float, default=0.05 4401 Maximum allowed p-value for statistical significance. If `adj=True`, 4402 this threshold is applied to the adjusted p-value (`adj_pval`); 4403 otherwise, it is applied to the raw p-value (`p_val`). 4404 4405 adj : bool, default=True 4406 Whether to use the adjusted p-value (`adj_pval`) instead of the raw 4407 p-value (`p_val`) for statistical significance filtering. 4408 4409 image_width : int or float, default=12 4410 Width of the regression plot in inches. 4411 4412 image_high : int or float, default=7 4413 Height of the regression plot in inches. 4414 4415 4416 4417 Returns 4418 ------- 4419 deg_stats : pandas.DataFrame 4420 DataFrame containing the DEG statistics after filtering by absolute 4421 log fold change, absolute ESM, and statistical significance. 4422 4423 fig : matplotlib.figure.Figure 4424 Matplotlib figure containing the regression plot, including expression 4425 points, regression line, regression statistics, and annotations for 4426 the selected top genes. 4427 4428 Raises 4429 ------ 4430 ValueError 4431 If `cell_x` or `cell_y` is not found in the dataset. 4432 4433 ValueError 4434 If multiple columns correspond to the same cell name and the required 4435 dataset identifiers (`set_x` and/or `set_y`) were not provided. 4436 4437 Notes 4438 ----- 4439 * The function calls `self.average()` before performing the analysis. 4440 * Linear regression is calculated using `scipy.stats.linregress`. 4441 * Regression residuals are calculated as the difference between the observed 4442 Y value and the value predicted by the fitted regression model. 4443 * Standardized residuals are used to prioritize genes that deviate most 4444 strongly from the expected linear relationship. 4445 * Significant genes are filtered using absolute log fold change, absolute 4446 ESM, and either raw or adjusted p-value. 4447 * Top genes are selected separately within each `valid_group`. 4448 * The top genes are annotated on the regression plot. 4449 * The returned `deg_stats` contains the significant genes before the final 4450 `n_max` per-group selection. 4451 4452 4453 Examples 4454 -------- 4455 > > > deg_stats, fig = obj.cell_regression( 4456 > > > ... cell_x="NeuronA", 4457 > > > ... cell_y="NeuronB", 4458 > > > ... min_log_min=0.5, 4459 > > > ... min_esm=0.5, 4460 > > > ... p_val=0.05, 4461 > > > ... n_max=20 4462 > > > ... ) 4463 > > > 4464 """ 4465 4466 self.average() 4467 4468 metadata = self.agg_metadata 4469 data = self.agg_normalized_data 4470 data2 = self.normalized_data 4471 metadata2 = self.input_metadata 4472 4473 if set_x is not None and set_y is not None: 4474 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4475 data2.columns = metadata2["cell_names"] + " # " + metadata2["sets"] 4476 cell_x = cell_x + " # " + set_x 4477 cell_y = cell_y + " # " + set_y 4478 4479 deg_stats = calc_DEG( 4480 data2, 4481 metadata_list=None, 4482 entities={ 4483 "cell_x": cell_x + " # " + set_x, 4484 "cell_y": cell_y + " # " + set_y, 4485 }, 4486 sets=None, 4487 min_exp=min_exp, 4488 min_pct=min_pct, 4489 n_proc=n_proc, 4490 ) 4491 4492 else: 4493 data.columns = metadata["cell_names"] 4494 data2.columns = metadata2["cell_names"] 4495 4496 deg_stats = calc_DEG( 4497 data2, 4498 metadata_list=None, 4499 entities={"cell_x": cell_x, "cell_y": cell_y}, 4500 sets=None, 4501 min_exp=min_exp, 4502 min_pct=min_pct, 4503 n_proc=n_proc, 4504 ) 4505 4506 # points 4507 x = data[cell_x] 4508 y = data[cell_y] 4509 4510 # regression 4511 slope, intercept, r_value, p_value, std_err = stats.linregress(x, y) 4512 4513 # prediction 4514 y_pred = slope * x + intercept 4515 4516 # residuals 4517 residual = y - y_pred 4518 4519 # residuals for DEG 4520 deg_stats["residual"] = deg_stats["feature"].map(residual) 4521 deg_stats["residual_abs"] = deg_stats["residual"].abs() 4522 deg_stats["residual_std"] = deg_stats["residual"] / residual.std() 4523 deg_stats["residual_std_abs"] = deg_stats["residual_std"].abs() 4524 deg_stats["esm_abs"] = deg_stats["esm"].abs() 4525 4526 deg_stats = deg_stats[deg_stats["log(FC)"].abs() >= min_log] 4527 4528 deg_stats = deg_stats[deg_stats["esm_abs"] >= min_esm] 4529 4530 if adj: 4531 4532 deg_stats = deg_stats[deg_stats["adj_pval"] <= p_val] 4533 else: 4534 deg_stats = deg_stats[deg_stats["p_val"] <= p_val] 4535 4536 deg_stats_n = ( 4537 deg_stats.sort_values( 4538 ["residual_std_abs", "esm_abs"], ascending=[False, False] 4539 ) 4540 .groupby("valid_group") 4541 .head(n_max) 4542 ) 4543 4544 if not cell_x in data.columns: 4545 raise ValueError("'cell_x' value not in cell names!") 4546 4547 if not cell_y in data.columns: 4548 raise ValueError("'cell_y' value not in cell names!") 4549 4550 if list(data.columns).count(cell_x) > 1: 4551 raise ValueError( 4552 f"'{cell_x}' occurs more than once. If you want to select a specific cell, " 4553 f"please also provide the corresponding 'set_x' and 'set_y' values." 4554 ) 4555 4556 if list(data.columns).count(cell_y) > 1: 4557 raise ValueError( 4558 f"'{cell_y}' occurs more than once. If you want to select a specific cell, " 4559 f"please also provide the corresponding 'set_x' and 'set_y' values." 4560 ) 4561 4562 data["color"] = "gray" 4563 4564 data.loc[data.index.isin(set(deg_stats["feature"])), "color"] = "green" 4565 4566 data.loc[data.index.isin(set(deg_stats_n["feature"])), "color"] = "blue" 4567 4568 color_order = {"gray": 0, "green": 1, "blue": 2} 4569 4570 data["color_order"] = data["color"].map(color_order) 4571 4572 data = data.sort_values("color_order") 4573 4574 fig, ax = plt.subplots(figsize=(image_width, image_high)) 4575 4576 # dots 4577 sns.scatterplot( 4578 x=cell_x, 4579 y=cell_y, 4580 data=data, 4581 hue="color", 4582 palette={"blue": "blue", "green": "green", "gray": "gray"}, 4583 legend=False, 4584 ax=ax, 4585 ) 4586 4587 # regression 4588 sns.regplot(x=cell_x, y=cell_y, data=data, scatter=False, color="red", ax=ax) 4589 4590 legend_elements = [ 4591 Line2D( 4592 [0], 4593 [0], 4594 marker="o", 4595 color="w", 4596 markerfacecolor="gray", 4597 markersize=7, 4598 label="Not significant", 4599 ), 4600 Line2D( 4601 [0], 4602 [0], 4603 marker="o", 4604 color="w", 4605 markerfacecolor="blue", 4606 markersize=7, 4607 label=f"Top {n_max}", 4608 ), 4609 Line2D( 4610 [0], 4611 [0], 4612 marker="o", 4613 color="w", 4614 markerfacecolor="green", 4615 markersize=7, 4616 label="Significant", 4617 ), 4618 Line2D([0], [0], color="red", linewidth=2, label="Linear regression"), 4619 ] 4620 4621 ax.legend(handles=legend_elements, loc="best", frameon=False) 4622 4623 slope, intercept, r_value, p_value, _ = stats.linregress( 4624 data[cell_x], data[cell_y] 4625 ) 4626 4627 equation = "y = {:.2f}x + {:.2f}".format(slope, intercept) 4628 4629 ax.annotate( 4630 "R-squared = {:.2f}\nP-value = {:.2f}\n{}".format( 4631 r_value**2, p_value, equation 4632 ), 4633 xy=(0.05, 0.90), 4634 xycoords="axes fraction", 4635 fontsize=12, 4636 ) 4637 4638 ax.spines["top"].set_visible(False) 4639 ax.spines["right"].set_visible(False) 4640 4641 # oznaczanie genów 4642 def annotate_outliers(x, y, features): 4643 texts = [] 4644 4645 for i, (xi, yi) in enumerate(zip(x, y)): 4646 if data.index[i] in features: 4647 text = ax.text(xi, yi, data.index[i]) 4648 texts.append(text) 4649 4650 return texts 4651 4652 features = list(set(deg_stats_n["feature"])) 4653 4654 texts = annotate_outliers(data[cell_x], data[cell_y], features) 4655 4656 adjust_text(texts, arrowprops=dict(arrowstyle="-", color="gray", alpha=0.5)) 4657 4658 return deg_stats, fig 4659 4660 def cell_genes_distribution( 4661 self, 4662 cells_names: list | None, 4663 cells_sets: list | None = None, 4664 image_width=10, 4665 image_high=7, 4666 ): 4667 """ 4668 Compare the distributions of the number of detected genes per cell across 4669 selected cell populations. 4670 4671 For each selected cell population, the function calculates the number of 4672 detected genes in each cell, defined as the number of genes with an expression 4673 value greater than zero. The resulting distributions are visualized as 4674 overlapping histograms, allowing the gene detection levels of different cell 4675 populations to be compared. 4676 4677 This analysis can be used as a quality-control step before comparing gene 4678 expression profiles or performing differential expression analysis. Similar 4679 distributions of detected genes per cell indicate comparable levels of gene 4680 detection across the selected populations, whereas substantial differences 4681 may indicate differences in sequencing depth, RNA content, or other technical 4682 factors. 4683 4684 Parameters 4685 ---------- 4686 cells_names : list 4687 List of cell population or cluster names to be included in the analysis. 4688 4689 cells_sets : list or None 4690 Optional list of dataset or experimental set identifiers corresponding 4691 element-wise to `cells_names`. This is required when the same cell 4692 population name occurs in multiple datasets and a specific dataset 4693 needs to be selected. 4694 4695 image_width : int or float, default=10 4696 Width of the histogram figure in inches. 4697 4698 image_high : int or float, default=7 4699 Height of the histogram figure in inches. 4700 4701 Returns 4702 ------- 4703 fig : matplotlib.figure.Figure 4704 Matplotlib figure containing overlapping histograms showing the 4705 distribution of the number of detected genes per cell for each selected 4706 cell population. 4707 4708 Raises 4709 ------ 4710 ValueError 4711 If `cells_names` is None or empty. 4712 4713 Notes 4714 ----- 4715 * A gene is considered detected when its expression value is greater than zero. 4716 * The number of detected genes is calculated independently for every cell. 4717 * Each selected cell population is represented by a separate histogram. 4718 * When `cells_sets` is provided and has the same length as `cells_names`, 4719 cell populations are selected using both their population name and dataset 4720 identifier. 4721 * The function operates on `self.normalized_data`. 4722 * The resulting distributions can be used to assess whether selected cell 4723 populations have comparable gene-detection levels before downstream 4724 comparisons. 4725 """ 4726 4727 metadata = self.input_metadata 4728 data = self.normalized_data 4729 4730 if cells_names is None or len(cells_names) == 0: 4731 raise ValueError("No cells selected!") 4732 4733 if cells_sets is not None and len(cells_sets) == len(cells_names): 4734 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4735 full_cells = [] 4736 for n in range(len(cells_names)): 4737 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4738 4739 data = data.loc[:, full_cells] 4740 4741 else: 4742 data.columns = metadata["cell_names"] 4743 data = data.loc[:, cells_names] 4744 4745 fig = plt.figure(figsize=(image_width, image_high)) 4746 4747 for name in data.columns.unique(): 4748 cols = data.columns == name 4749 4750 counts = (data.loc[:, cols] > 0).sum(axis=0) 4751 4752 plt.hist(counts, bins=30, alpha=0.5, label=name) 4753 4754 plt.xlabel("Number of detected genes per cell") 4755 plt.ylabel("Number of cells") 4756 plt.legend() 4757 plt.tight_layout() 4758 4759 return fig 4760 4761 def mm_cell_genes( 4762 self, 4763 cells_names: list | None, 4764 cells_sets: list | None, 4765 non_include_zeros: bool = True, 4766 image_width=7, 4767 image_high=10, 4768 ): 4769 """ 4770 Compare the mean and median gene expression across selected cell populations. 4771 4772 For each selected cell population, the function calculates the mean and median 4773 expression values across all genes and cells. By default, zero-expression 4774 values are excluded from the calculation, allowing the comparison to focus on 4775 genes with detected expression. When `non_include_zeros` is set to False, zero 4776 values are included in the calculation. 4777 4778 The resulting mean and median expression values are visualized as a grouped 4779 bar plot, with separate bars representing the mean and median for each cell 4780 population. This visualization can be used to assess differences in the overall 4781 expression level and distribution between selected cell populations. 4782 4783 Parameters 4784 ---------- 4785 cells_names : list 4786 List of cell population or cluster names to be included in the analysis. 4787 4788 cells_sets : list or None 4789 Optional list of dataset or experimental set identifiers corresponding 4790 element-wise to `cells_names`. When provided and its length matches 4791 `cells_names`, cell populations are selected using both their population 4792 name and dataset identifier. 4793 4794 non_include_zeros : bool, default=True 4795 Whether to exclude zero-expression values before calculating the mean and 4796 median. If True, only non-zero expression values are used. If False, all 4797 expression values, including zeros, are used. 4798 4799 image_width : int or float, default=7 4800 Width of the figure in inches. 4801 4802 image_high : int or float, default=10 4803 Height of the figure in inches. 4804 4805 Returns 4806 ------- 4807 fig : matplotlib.figure.Figure 4808 Matplotlib figure containing grouped bar plots showing the mean and median 4809 expression for each selected cell population. 4810 4811 Raises 4812 ------ 4813 ValueError 4814 If `cells_names` is None or empty. 4815 4816 Notes 4817 ----- 4818 * Expression values are obtained from `self.normalized_data`. 4819 * The mean and median are calculated independently for each selected cell 4820 population. 4821 * When `non_include_zeros=True`, zero-expression values are excluded before 4822 calculating both statistics. 4823 * When `non_include_zeros=False`, zero-expression values contribute to both 4824 the mean and median. 4825 * The analysis uses all genes and cells belonging to each selected population. 4826 * The resulting statistics describe the overall distribution of expression 4827 values and can be used to compare the expression levels of different cell 4828 populations. 4829 """ 4830 4831 metadata = self.input_metadata 4832 data = self.normalized_data 4833 4834 if cells_names is None or len(cells_names) == 0: 4835 raise ValueError("No cells selected!") 4836 4837 if cells_sets is not None and len(cells_sets) == len(cells_names): 4838 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4839 full_cells = [] 4840 for n in range(len(cells_names)): 4841 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4842 4843 data = data.loc[:, full_cells] 4844 4845 else: 4846 data.columns = metadata["cell_names"] 4847 data = data.loc[:, cells_names] 4848 4849 groups = data.columns.unique() 4850 4851 fig = plt.figure(figsize=(image_width, image_high)) 4852 4853 mean_expression = {} 4854 median_expression = {} 4855 4856 for name in groups: 4857 cols = data.columns == name 4858 4859 values = data.loc[:, cols].values.flatten() 4860 4861 if non_include_zeros: 4862 values = values[values != 0] 4863 4864 mean_expression[name] = values.mean() 4865 median_expression[name] = np.median(values) 4866 4867 mean_expression = pd.Series(mean_expression) 4868 median_expression = pd.Series(median_expression) 4869 4870 x = np.arange(len(groups)) 4871 width = 0.35 4872 4873 plt.bar(x - width / 2, mean_expression.values, width, label="Mean") 4874 4875 plt.bar(x + width / 2, median_expression.values, width, label="Median") 4876 4877 plt.xlabel("Cell") 4878 plt.ylabel("Expression") 4879 plt.xticks(x, groups, rotation=45) 4880 plt.legend() 4881 4882 plt.tight_layout() 4883 4884 return fig 4885 4886 def cell_dispersion( 4887 self, 4888 cells_names: list | None, 4889 cells_sets: list | None, 4890 image_width=7, 4891 image_high=10, 4892 ): 4893 """ 4894 Assess the within-cluster dispersion of cell gene expression profiles. 4895 4896 For each selected cell population, the function calculates the centroid of 4897 the population as the mean expression profile across all cells in the cluster. 4898 The Euclidean distance between each individual cell and its corresponding 4899 cluster centroid is then calculated to quantify how much the cell's expression 4900 profile deviates from the average profile of the cluster. 4901 4902 The resulting distributions of distances are visualized using violin plots, 4903 with each violin representing one selected cell population. The distribution 4904 shows the degree of transcriptional heterogeneity within each cluster: 4905 smaller distances indicate cells with more similar expression profiles, 4906 whereas larger distances indicate greater within-cluster variability. 4907 4908 This analysis can be used to assess the internal consistency and transcriptional 4909 heterogeneity of cell populations before comparing their expression profiles 4910 between clusters. 4911 4912 Parameters 4913 ---------- 4914 self : object 4915 self object containing the input metadata and normalized expression 4916 data. 4917 4918 cells_names : list 4919 List of cell population or cluster names to be included in the analysis. 4920 4921 cells_sets : list or None 4922 Optional list of dataset or experimental set identifiers corresponding 4923 element-wise to `cells_names`. When provided and its length matches 4924 `cells_names`, cell populations are selected using both their population 4925 name and dataset identifier. 4926 4927 image_width : int or float, default=7 4928 Width of the figure in inches. 4929 4930 image_high : int or float, default=10 4931 Height of the figure in inches. 4932 4933 Returns 4934 ------- 4935 fig : matplotlib.figure.Figure 4936 Matplotlib figure containing violin plots showing the distribution of 4937 Euclidean distances between individual cells and their respective cluster 4938 centroids. 4939 4940 Raises 4941 ------ 4942 ValueError 4943 If `cells_names` is None or empty. 4944 4945 Notes 4946 ----- 4947 * Expression values are obtained from `self.normalized_data`. 4948 * A cluster centroid is calculated as the mean expression profile across all 4949 cells belonging to that cluster. 4950 * The distance of each cell from the centroid is calculated using Euclidean 4951 distance across all genes. 4952 * Each cell contributes one distance value to the distribution of its cluster. 4953 * Smaller distances indicate lower within-cluster transcriptional dispersion. 4954 * Larger distances indicate greater within-cluster transcriptional dispersion. 4955 * The violin plot includes a box plot showing the median and interquartile 4956 range of the distance distribution. 4957 * Differences in dispersion between clusters can be used to assess their 4958 relative transcriptional homogeneity before downstream comparisons. 4959 """ 4960 4961 metadata = self.input_metadata 4962 data = self.normalized_data 4963 4964 if cells_names is None or len(cells_names) == 0: 4965 raise ValueError("No cells selected!") 4966 4967 if cells_sets is not None and len(cells_sets) == len(cells_names): 4968 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4969 full_cells = [] 4970 for n in range(len(cells_names)): 4971 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4972 4973 data = data.loc[:, full_cells] 4974 4975 else: 4976 data.columns = metadata["cell_names"] 4977 data = data.loc[:, cells_names] 4978 4979 fig = plt.figure(figsize=(image_width, image_high)) 4980 4981 dispersion_long = [] 4982 4983 for cluster in data.columns.unique(): 4984 4985 cells = data.loc[:, data.columns == cluster] 4986 4987 centroid = cells.mean(axis=1) 4988 4989 distances = np.sqrt(((cells.sub(centroid, axis=0)) ** 2).sum(axis=0)) 4990 4991 for cell, distance in distances.items(): 4992 dispersion_long.append( 4993 {"cluster": cluster, "cell": cell, "distance_to_centroid": distance} 4994 ) 4995 4996 dispersion_long = pd.DataFrame(dispersion_long) 4997 4998 sns.violinplot( 4999 data=dispersion_long, x="cluster", y="distance_to_centroid", inner="box" 5000 ) 5001 5002 plt.ylabel("Distance from cluster centroid") 5003 plt.xlabel("Cluster") 5004 plt.xticks(rotation=45) 5005 plt.tight_layout() 5006 5007 return fig
class
Clustering:
33class Clustering: 34 """ 35 A class for performing dimensionality reduction, clustering, and visualization 36 on high-dimensional data (e.g., single-cell gene expression). 37 38 The class provides methods for: 39 - Normalizing and extracting subsets of data 40 - Principal Component Analysis (PCA) and related clustering 41 - Uniform Manifold Approximation and Projection (UMAP) and clustering 42 - Visualization of PCA and UMAP embeddings 43 - Harmonization of batch effects 44 - Accessing processed data and cluster labels 45 46 Methods 47 ------- 48 add_data_frame(data, metadata) 49 Class method to create a Clustering instance from a DataFrame and metadata. 50 51 harmonize_sets() 52 Perform batch effect harmonization on PCA data. 53 54 perform_PCA(pc_num=100, width=8, height=6) 55 Perform PCA on the dataset and visualize the first two PCs. 56 57 knee_plot_PCA(width=8, height=6) 58 Plot the cumulative variance explained by PCs to determine optimal dimensionality. 59 60 find_clusters_PCA(pc_num=0, eps=0.5, min_samples=10, width=8, height=6, harmonized=False) 61 Apply DBSCAN clustering to PCA embeddings and visualize results. 62 63 perform_UMAP(factorize=False, umap_num=100, pc_num=0, harmonized=False, ...) 64 Compute UMAP embeddings with optional parameter tuning. 65 66 knee_plot_umap(eps=0.5, min_samples=10) 67 Determine optimal UMAP dimensionality using silhouette scores. 68 69 find_clusters_UMAP(umap_n=5, eps=0.5, min_samples=10, width=8, height=6) 70 Apply DBSCAN clustering on UMAP embeddings and visualize clusters. 71 72 UMAP_vis(names_slot='cell_names', set_sep=True, point_size=0.6, ...) 73 Visualize UMAP embeddings with labels and optional cluster numbering. 74 75 UMAP_feature(feature_name, features_data=None, point_size=0.6, ...) 76 Plot a single feature over UMAP coordinates with customizable colormap. 77 78 get_umap_data() 79 Return the UMAP embeddings along with cluster labels if available. 80 81 get_pca_data() 82 Return the PCA results along with cluster labels if available. 83 84 return_clusters(clusters='umap') 85 Return the cluster labels for UMAP or PCA embeddings. 86 87 Raises 88 ------ 89 ValueError 90 For invalid parameters, mismatched dimensions, or missing metadata. 91 """ 92 93 def __init__(self, data, metadata): 94 """ 95 Initialize the clustering class with data and optional metadata. 96 97 Parameters 98 ---------- 99 data : pandas.DataFrame 100 Input data for clustering. Columns are considered as samples. 101 102 metadata : pandas.DataFrame, optional 103 Metadata for the samples. If None, a default DataFrame with column 104 names as 'cell_names' is created. 105 106 Attributes 107 ---------- 108 -clustering_data : pandas.DataFrame 109 -clustering_metadata : pandas.DataFrame 110 -subclusters : None or dict 111 -explained_var : None or numpy.ndarray 112 -cumulative_var : None or numpy.ndarray 113 -pca : None or pandas.DataFrame 114 -harmonized_pca : None or pandas.DataFrame 115 -umap : None or pandas.DataFrame 116 """ 117 118 self.clustering_data = data 119 """The input data used for clustering.""" 120 121 if metadata is None: 122 metadata = pd.DataFrame({"cell_names": list(data.columns)}) 123 124 self.clustering_metadata = metadata 125 """Metadata associated with the samples.""" 126 127 self.subclusters = None 128 """Placeholder for storing subcluster information.""" 129 130 self.explained_var = None 131 """Explained variance from PCA, initialized as None.""" 132 133 self.cumulative_var = None 134 """Cumulative explained variance from PCA, initialized as None.""" 135 136 self.pca = None 137 """PCA-transformed data, initialized as None.""" 138 139 self.harmonized_pca = None 140 """PCA data after batch effect harmonization, initialized as None.""" 141 142 self.umap = None 143 """UMAP embeddings, initialized as None.""" 144 145 @classmethod 146 def add_data_frame(cls, data: pd.DataFrame, metadata: pd.DataFrame | None): 147 """ 148 Create a Clustering instance from a DataFrame and optional metadata. 149 150 Parameters 151 ---------- 152 data : pandas.DataFrame 153 Input data with features as rows and samples/cells as columns. 154 155 metadata : pandas.DataFrame or None 156 Optional metadata for the samples. 157 Each row corresponds to a sample/cell, and column names in this DataFrame 158 should match the sample/cell names in `data`. Columns can contain additional 159 information such as cell type, experimental condition, batch, sets, etc. 160 161 Returns 162 ------- 163 Clustering 164 A new instance of the Clustering class. 165 """ 166 167 return cls(data, metadata) 168 169 def harmonize_sets(self, batch_col: str = "sets"): 170 """ 171 Perform batch effect harmonization on PCA embeddings. 172 173 Parameters 174 ---------- 175 batch_col : str, default 'sets' 176 Name of the column in `metadata` that contains batch information for the samples/cells. 177 178 Returns 179 ------- 180 None 181 Updates the `harmonized_pca` attribute with harmonized data. 182 """ 183 184 data_mat = np.array(self.pca) 185 186 metadata = self.clustering_metadata 187 188 self.harmonized_pca = pd.DataFrame( 189 harmonize.run_harmony(data_mat, metadata, vars_use=batch_col).Z_corr 190 ).T 191 192 self.harmonized_pca.columns = self.pca.columns 193 194 def perform_PCA(self, pc_num: int = 100, width=8, height=6): 195 """ 196 Perform Principal Component Analysis (PCA) on the dataset. 197 198 This method standardizes the data, applies PCA, stores results as attributes, 199 and generates a scatter plot of the first two principal components. 200 201 Parameters 202 ---------- 203 pc_num : int, default 100 204 Number of principal components to compute. 205 If 0, computes all available components. 206 207 width : int or float, default 8 208 Width of the PCA figure. 209 210 height : int or float, default 6 211 Height of the PCA figure. 212 213 Returns 214 ------- 215 matplotlib.figure.Figure 216 Scatter plot showing the first two principal components. 217 218 Updates 219 ------- 220 self.pca : pandas.DataFrame 221 DataFrame with principal component scores for each sample. 222 223 self.explained_var : numpy.ndarray 224 Percentage of variance explained by each principal component. 225 226 self.cumulative_var : numpy.ndarray 227 Cumulative explained variance. 228 """ 229 230 scaler = StandardScaler() 231 data_scaled = scaler.fit_transform(self.clustering_data.T) 232 233 if pc_num == 0 or pc_num > data_scaled.shape[0]: 234 pc_num = data_scaled.shape[0] 235 236 pca = PCA(n_components=pc_num, random_state=42) 237 238 principal_components = pca.fit_transform(data_scaled) 239 240 pca_df = pd.DataFrame( 241 data=principal_components, 242 columns=["PC" + str(x + 1) for x in range(pc_num)], 243 ) 244 245 self.explained_var = pca.explained_variance_ratio_ * 100 246 self.cumulative_var = np.cumsum(self.explained_var) 247 248 self.pca = pca_df 249 250 fig = plt.figure(figsize=(width, height)) 251 plt.scatter(pca_df["PC1"], pca_df["PC2"], alpha=0.7) 252 plt.xlabel("PC 1") 253 plt.ylabel("PC 2") 254 plt.grid(True) 255 plt.show() 256 257 return fig 258 259 def knee_plot_PCA(self, width: int = 8, height: int = 6): 260 """ 261 Plot cumulative explained variance to determine the optimal number of PCs. 262 263 Parameters 264 ---------- 265 width : int, default 8 266 Width of the figure. 267 268 height : int or, default 6 269 Height of the figure. 270 271 Returns 272 ------- 273 matplotlib.figure.Figure 274 Line plot showing cumulative variance explained by each PC. 275 """ 276 277 fig_knee = plt.figure(figsize=(width, height)) 278 plt.plot(range(1, len(self.explained_var) + 1), self.cumulative_var, marker="o") 279 plt.xlabel("PC (n components)") 280 plt.ylabel("Cumulative explained variance (%)") 281 plt.grid(True) 282 283 xticks = [1] + list(range(5, len(self.explained_var) + 1, 5)) 284 plt.xticks(xticks, rotation=60) 285 286 plt.show() 287 288 return fig_knee 289 290 def find_clusters_PCA( 291 self, 292 pc_num: int = 2, 293 eps: float = 0.5, 294 min_samples: int = 10, 295 width: int = 8, 296 height: int = 6, 297 harmonized: bool = False, 298 ): 299 """ 300 Apply DBSCAN clustering to PCA embeddings and visualize the results. 301 302 This method performs density-based clustering (DBSCAN) on the PCA-reduced 303 dataset. Cluster labels are stored in the object's metadata, and a scatter 304 plot of the first two principal components with cluster annotations is returned. 305 306 Parameters 307 ---------- 308 pc_num : int, default 2 309 Number of principal components to use for clustering. 310 If 0, uses all available components. 311 312 eps : float, default 0.5 313 Maximum distance between two points for them to be considered 314 as neighbors (DBSCAN parameter). 315 316 min_samples : int, default 10 317 Minimum number of samples required to form a cluster (DBSCAN parameter). 318 319 width : int, default 8 320 Width of the output scatter plot. 321 322 height : int, default 6 323 Height of the output scatter plot. 324 325 harmonized : bool, default False 326 If True, use harmonized PCA data (`self.harmonized_pca`). 327 If False, use standard PCA results (`self.pca`). 328 329 Returns 330 ------- 331 matplotlib.figure.Figure 332 Scatter plot of the first two principal components colored by 333 cluster assignments. 334 335 Updates 336 ------- 337 self.clustering_metadata['PCA_clusters'] : list 338 Cluster labels assigned to each cell/sample. 339 340 self.input_metadata['PCA_clusters'] : list, optional 341 Cluster labels stored in input metadata (if available). 342 """ 343 344 dbscan = DBSCAN(eps=eps, min_samples=min_samples) 345 346 if pc_num == 0 and harmonized: 347 PCA = self.harmonized_pca 348 349 elif pc_num == 0: 350 PCA = self.pca 351 352 else: 353 if harmonized: 354 355 PCA = self.harmonized_pca.iloc[:, 0:pc_num] 356 else: 357 358 PCA = self.pca.iloc[:, 0:pc_num] 359 360 dbscan_labels = dbscan.fit_predict(PCA) 361 362 pca_df = pd.DataFrame(PCA) 363 pca_df["Cluster"] = dbscan_labels 364 365 fig = plt.figure(figsize=(width, height)) 366 367 for cluster_id in sorted(pca_df["Cluster"].unique()): 368 cluster_data = pca_df[pca_df["Cluster"] == cluster_id] 369 plt.scatter( 370 cluster_data["PC1"], 371 cluster_data["PC2"], 372 label=f"Cluster {cluster_id}", 373 alpha=0.7, 374 ) 375 376 plt.xlabel("PC 1") 377 plt.ylabel("PC 2") 378 plt.legend(title="Clusters", loc="center left", bbox_to_anchor=(1.0, 0.5)) 379 380 plt.grid(True) 381 plt.show() 382 383 self.clustering_metadata["PCA_clusters"] = [str(x) for x in dbscan_labels] 384 385 try: 386 self.input_metadata["PCA_clusters"] = [str(x) for x in dbscan_labels] 387 except: 388 pass 389 390 return fig 391 392 def perform_UMAP( 393 self, 394 factorize: bool = False, 395 umap_num: int = 100, 396 pc_num: int = 0, 397 harmonized: bool = False, 398 n_neighbors: int = 5, 399 min_dist: float | int = 0.1, 400 spread: float | int = 1.0, 401 set_op_mix_ratio: float | int = 1.0, 402 local_connectivity: int = 1, 403 repulsion_strength: float | int = 1.0, 404 negative_sample_rate: int = 5, 405 width: int = 8, 406 height: int = 6, 407 ): 408 """ 409 Compute and visualize UMAP embeddings of the dataset. 410 411 This method applies Uniform Manifold Approximation and Projection (UMAP) 412 for dimensionality reduction on either raw, PCA, or harmonized PCA data. 413 Results are stored as a DataFrame (`self.umap`) and a scatter plot figure 414 (`self.UMAP_plot`). 415 416 Parameters 417 ---------- 418 factorize : bool, default False 419 If True, categorical sample labels (from column names) are factorized 420 and used as supervision in UMAP fitting. 421 422 umap_num : int, default 100 423 Number of UMAP dimensions to compute. If 0, matches the input dimension. 424 425 pc_num : int, default 0 426 Number of principal components to use as UMAP input. 427 If 0, use all available components or raw data. 428 429 harmonized : bool, default False 430 If True, use harmonized PCA embeddings (`self.harmonized_pca`). 431 If False, use standard PCA or raw scaled data. 432 433 n_neighbors : int, default 5 434 UMAP parameter controlling the size of the local neighborhood. 435 436 min_dist : float, default 0.1 437 UMAP parameter controlling minimum allowed distance between embedded points. 438 439 spread : int | float, default 1.0 440 Effective scale of embedded space (UMAP parameter). 441 442 set_op_mix_ratio : int | float, default 1.0 443 Interpolation parameter between union and intersection in fuzzy sets. 444 445 local_connectivity : int, default 1 446 Number of nearest neighbors assumed for each point. 447 448 repulsion_strength : int | float, default 1.0 449 Weighting applied to negative samples during optimization. 450 451 negative_sample_rate : int, default 5 452 Number of negative samples per positive sample in optimization. 453 454 width : int, default 8 455 Width of the output scatter plot. 456 457 height : int, default 6 458 Height of the output scatter plot. 459 460 Updates 461 ------- 462 self.umap : pandas.DataFrame 463 Table of UMAP embeddings with columns `UMAP1 ... UMAPn`. 464 465 Notes 466 ----- 467 For supervised UMAP (`factorize=True`), categorical codes from column 468 names of the dataset are used as labels. 469 """ 470 471 scaler = StandardScaler() 472 473 if pc_num == 0 and harmonized: 474 data_scaled = self.harmonized_pca 475 476 elif pc_num == 0: 477 data_scaled = scaler.fit_transform(self.clustering_data.T) 478 479 else: 480 if harmonized: 481 482 data_scaled = self.harmonized_pca.iloc[:, 0:pc_num] 483 else: 484 485 data_scaled = self.pca.iloc[:, 0:pc_num] 486 487 if umap_num == 0 or umap_num > data_scaled.shape[1]: 488 489 umap_num = data_scaled.shape[1] 490 491 reducer = umap.UMAP( 492 n_components=len(data_scaled.T), 493 random_state=42, 494 n_neighbors=n_neighbors, 495 min_dist=min_dist, 496 spread=spread, 497 set_op_mix_ratio=set_op_mix_ratio, 498 local_connectivity=local_connectivity, 499 repulsion_strength=repulsion_strength, 500 negative_sample_rate=negative_sample_rate, 501 n_jobs=1, 502 ) 503 504 else: 505 506 reducer = umap.UMAP( 507 n_components=umap_num, 508 random_state=42, 509 n_neighbors=n_neighbors, 510 min_dist=min_dist, 511 spread=spread, 512 set_op_mix_ratio=set_op_mix_ratio, 513 local_connectivity=local_connectivity, 514 repulsion_strength=repulsion_strength, 515 negative_sample_rate=negative_sample_rate, 516 n_jobs=1, 517 ) 518 519 if factorize: 520 embedding = reducer.fit_transform( 521 X=data_scaled, y=pd.Categorical(self.clustering_data.columns).codes 522 ) 523 else: 524 embedding = reducer.fit_transform(X=data_scaled) 525 526 umap_df = pd.DataFrame( 527 embedding, columns=["UMAP" + str(x + 1) for x in range(umap_num)] 528 ) 529 530 plt.figure(figsize=(width, height)) 531 plt.scatter(umap_df["UMAP1"], umap_df["UMAP2"], alpha=0.7) 532 plt.xlabel("UMAP 1") 533 plt.ylabel("UMAP 2") 534 plt.grid(True) 535 536 plt.show() 537 538 self.umap = umap_df 539 540 def knee_plot_umap(self, eps: int | float = 0.5, min_samples: int = 10): 541 """ 542 Plot silhouette scores for different UMAP dimensions to determine optimal n_components. 543 544 Parameters 545 ---------- 546 eps : float, default 0.5 547 DBSCAN eps parameter for clustering each UMAP dimension. 548 549 min_samples : int, default 10 550 Minimum number of samples to form a cluster in DBSCAN. 551 552 Returns 553 ------- 554 matplotlib.figure.Figure 555 Silhouette score plot across UMAP dimensions. 556 """ 557 558 umap_range = range(2, len(self.umap.T) + 1) 559 560 silhouette_scores = [] 561 component = [] 562 for n in umap_range: 563 564 db = DBSCAN(eps=eps, min_samples=min_samples) 565 labels = db.fit_predict(np.array(self.umap)[:, :n]) 566 567 mask = labels != -1 568 if len(set(labels[mask])) > 1: 569 score = silhouette_score(np.array(self.umap)[:, :n][mask], labels[mask]) 570 else: 571 score = -1 572 573 silhouette_scores.append(score) 574 component.append(n) 575 576 fig = plt.figure(figsize=(10, 5)) 577 plt.plot(component, silhouette_scores, marker="o") 578 plt.xlabel("UMAP (n_components)") 579 plt.ylabel("Silhouette Score") 580 plt.grid(True) 581 plt.xticks(range(int(min(component)), int(max(component)) + 1, 1)) 582 583 plt.show() 584 585 return fig 586 587 def find_clusters_UMAP( 588 self, 589 umap_n: int = 5, 590 eps: float | float = 0.5, 591 min_samples: int = 10, 592 width: int = 8, 593 height: int = 6, 594 ): 595 """ 596 Apply DBSCAN clustering on UMAP embeddings and visualize clusters. 597 598 This method performs density-based clustering (DBSCAN) on the UMAP-reduced 599 dataset. Cluster labels are stored in the object's metadata, and a scatter 600 plot of the first two UMAP components with cluster annotations is returned. 601 602 Parameters 603 ---------- 604 umap_n : int, default 5 605 Number of UMAP dimensions to use for DBSCAN clustering. 606 Must be <= number of columns in `self.umap`. 607 608 eps : float | int, default 0.5 609 Maximum neighborhood distance between two samples for them to be considered 610 as in the same cluster (DBSCAN parameter). 611 612 min_samples : int, default 10 613 Minimum number of samples in a neighborhood to form a cluster (DBSCAN parameter). 614 615 width : int, default 8 616 Figure width. 617 618 height : int, default 6 619 Figure height. 620 621 Returns 622 ------- 623 matplotlib.figure.Figure 624 Scatter plot of the first two UMAP components colored by 625 cluster assignments. 626 627 Updates 628 ------- 629 self.clustering_metadata['UMAP_clusters'] : list 630 Cluster labels assigned to each cell/sample. 631 632 self.input_metadata['UMAP_clusters'] : list, optional 633 Cluster labels stored in input metadata (if available). 634 """ 635 636 dbscan = DBSCAN(eps=eps, min_samples=min_samples) 637 dbscan_labels = dbscan.fit_predict(np.array(self.umap)[:, :umap_n]) 638 639 umap_df = self.umap 640 umap_df["Cluster"] = dbscan_labels 641 642 fig = plt.figure(figsize=(width, height)) 643 644 for cluster_id in sorted(umap_df["Cluster"].unique()): 645 cluster_data = umap_df[umap_df["Cluster"] == cluster_id] 646 plt.scatter( 647 cluster_data["UMAP1"], 648 cluster_data["UMAP2"], 649 label=f"Cluster {cluster_id}", 650 alpha=0.7, 651 ) 652 653 plt.xlabel("UMAP 1") 654 plt.ylabel("UMAP 2") 655 plt.legend(title="Clusters", loc="center left", bbox_to_anchor=(1.0, 0.5)) 656 plt.grid(True) 657 658 self.clustering_metadata["UMAP_clusters"] = [str(x) for x in dbscan_labels] 659 660 try: 661 self.input_metadata["UMAP_clusters"] = [str(x) for x in dbscan_labels] 662 except: 663 pass 664 665 return fig 666 667 def UMAP_vis( 668 self, 669 names_slot: str = "cell_names", 670 set_sep: bool = True, 671 point_size: int | float = 0.6, 672 font_size: int | float = 6, 673 legend_split_col: int = 2, 674 width: int = 8, 675 height: int = 6, 676 inc_num: bool = True, 677 ): 678 """ 679 Visualize UMAP embeddings with sample labels based on specyfic metadata slot. 680 681 Parameters 682 ---------- 683 names_slot : str, default 'cell_names' 684 Column in metadata to use as sample labels. 685 686 set_sep : bool, default True 687 If True, separate points by dataset. 688 689 point_size : float, default 0.6 690 Size of scatter points. 691 692 font_size : int, default 6 693 Font size for numbers on points. 694 695 legend_split_col : int, default 2 696 Number of columns in legend. 697 698 width : int, default 8 699 Figure width. 700 701 height : int, default 6 702 Figure height. 703 704 inc_num : bool, default True 705 If True, annotate points with numeric labels. 706 707 Returns 708 ------- 709 matplotlib.figure.Figure 710 UMAP scatter plot figure. 711 """ 712 713 umap_df = self.umap.iloc[:, 0:2].copy() 714 umap_df["names"] = list(self.clustering_metadata[names_slot]) 715 716 if set_sep: 717 718 if "sets" in list(self.clustering_metadata.columns): 719 umap_df["dataset"] = list(self.clustering_metadata["sets"]) 720 else: 721 umap_df["dataset"] = "default" 722 723 else: 724 umap_df["dataset"] = "default" 725 726 umap_df["tmp_nam"] = list(umap_df["names"] + umap_df["dataset"]) 727 728 umap_df["count"] = umap_df["tmp_nam"].map(umap_df["tmp_nam"].value_counts()) 729 730 numeric_df = ( 731 pd.DataFrame(umap_df[["count", "tmp_nam", "names"]].copy()) 732 .drop_duplicates() 733 .sort_values("count", ascending=False) 734 ) 735 numeric_df["numeric_values"] = range(0, numeric_df.shape[0]) 736 737 umap_df = umap_df.merge( 738 numeric_df[["tmp_nam", "numeric_values"]], on="tmp_nam", how="left" 739 ) 740 741 fig, ax = plt.subplots(figsize=(width, height)) 742 743 markers = ["o", "s", "^", "D", "P", "*", "X"] 744 marker_map = { 745 ds: markers[i % len(markers)] 746 for i, ds in enumerate(umap_df["dataset"].unique()) 747 } 748 749 cord_list = [] 750 751 for num, nam in zip(numeric_df["numeric_values"], numeric_df["names"]): 752 753 cluster_data = umap_df[umap_df["numeric_values"] == num] 754 755 ax.scatter( 756 cluster_data["UMAP1"], 757 cluster_data["UMAP2"], 758 label=f"{num} - {nam}", 759 marker=marker_map[cluster_data["dataset"].iloc[0]], 760 alpha=0.6, 761 s=point_size, 762 ) 763 764 coords = cluster_data[["UMAP1", "UMAP2"]].values 765 766 dists = pairwise_distances(coords) 767 768 sum_dists = dists.sum(axis=1) 769 770 center_idx = np.argmin(sum_dists) 771 center_point = coords[center_idx] 772 773 cord_list.append(center_point) 774 775 if inc_num: 776 texts = [] 777 for (x, y), num in zip(cord_list, numeric_df["numeric_values"]): 778 texts.append( 779 ax.text( 780 x, 781 y, 782 str(num), 783 ha="center", 784 va="center", 785 fontsize=font_size, 786 color="black", 787 ) 788 ) 789 790 adjust_text(texts, arrowprops=dict(arrowstyle="-", color="gray", lw=0.5)) 791 792 ax.set_xlabel("UMAP 1") 793 ax.set_ylabel("UMAP 2") 794 795 ax.legend( 796 title="Clusters", 797 loc="center left", 798 bbox_to_anchor=(1.05, 0.5), 799 ncol=legend_split_col, 800 markerscale=5, 801 ) 802 803 ax.grid(True) 804 805 plt.tight_layout() 806 807 return fig 808 809 def UMAP_feature( 810 self, 811 feature_name: str, 812 features_data: pd.DataFrame | None, 813 point_size: int | float = 0.6, 814 font_size: int | float = 6, 815 width: int = 8, 816 height: int = 6, 817 palette="light", 818 ): 819 """ 820 Visualize UMAP embedding with expression levels of a selected feature. 821 822 Each point (cell) in the UMAP plot is colored according to the expression 823 value of the chosen feature, enabling interpretation of spatial patterns 824 of gene activity or metadata distribution in low-dimensional space. 825 826 Parameters 827 ---------- 828 feature_name : str 829 Name of the feature to plot. 830 831 features_data : pandas.DataFrame or None, default None 832 If None, the function uses the DataFrame containing the clustering data. 833 To plot features not used in clustering, provide a wider DataFrame 834 containing the original feature values. 835 836 point_size : float, default 0.6 837 Size of scatter points in the plot. 838 839 font_size : int, default 6 840 Font size for axis labels and annotations. 841 842 width : int, default 8 843 Width of the matplotlib figure. 844 845 height : int, default 6 846 Height of the matplotlib figure. 847 848 palette : str, default 'light' 849 Color palette for expression visualization. Options are: 850 - 'light' 851 - 'dark' 852 - 'green' 853 - 'gray' 854 855 Returns 856 ------- 857 matplotlib.figure.Figure 858 UMAP scatter plot colored by feature values. 859 """ 860 861 umap_df = self.umap.iloc[:, 0:2].copy() 862 863 if features_data is None: 864 865 features_data = self.clustering_data 866 867 if features_data.shape[1] != umap_df.shape[0]: 868 raise ValueError( 869 "Imputed 'features_data' shape does not match the number of UMAP cells" 870 ) 871 872 blist = [ 873 True if x.upper() == feature_name.upper() else False 874 for x in features_data.index 875 ] 876 877 if not any(blist): 878 raise ValueError("Imputed feature_name is not included in the data") 879 880 umap_df.loc[:, "feature"] = ( 881 features_data.loc[blist, :] 882 .apply(lambda row: row.tolist(), axis=1) 883 .values[0] 884 ) 885 886 umap_df = umap_df.sort_values("feature", ascending=True) 887 888 import matplotlib.colors as mcolors 889 890 if palette == "light": 891 palette = px.colors.sequential.Sunsetdark 892 893 elif palette == "dark": 894 palette = px.colors.sequential.thermal 895 896 elif palette == "green": 897 palette = px.colors.sequential.Aggrnyl 898 899 elif palette == "gray": 900 palette = px.colors.sequential.gray 901 palette = palette[::-1] 902 903 else: 904 raise ValueError( 905 'Palette not found. Use: "light", "dark", "gray", or "green"' 906 ) 907 908 converted = [] 909 for c in palette: 910 rgb_255 = px.colors.unlabel_rgb(c) 911 rgb_01 = tuple(v / 255.0 for v in rgb_255) 912 converted.append(rgb_01) 913 914 my_cmap = mcolors.ListedColormap(converted, name="custom") 915 916 fig, ax = plt.subplots(figsize=(width, height)) 917 sc = ax.scatter( 918 umap_df["UMAP1"], 919 umap_df["UMAP2"], 920 c=umap_df["feature"], 921 s=point_size, 922 cmap=my_cmap, 923 alpha=1.0, 924 edgecolors="black", 925 linewidths=0.1, 926 ) 927 928 cbar = plt.colorbar(sc, ax=ax) 929 cbar.set_label(f"{feature_name}") 930 931 ax.set_xlabel("UMAP 1") 932 ax.set_ylabel("UMAP 2") 933 934 ax.grid(True) 935 936 plt.tight_layout() 937 938 return fig 939 940 def get_umap_data(self): 941 """ 942 Retrieve UMAP embedding data with optional cluster labels. 943 944 Returns the UMAP coordinates stored in `self.umap`. If clustering 945 metadata is available (specifically `UMAP_clusters`), the corresponding 946 cluster assignments are appended as an additional column. 947 948 Returns 949 ------- 950 pandas.DataFrame 951 DataFrame containing UMAP coordinates (columns: 'UMAP1', 'UMAP2', ...). 952 If available, includes an extra column 'clusters' with cluster labels. 953 954 Notes 955 ----- 956 - UMAP embeddings must be computed beforehand (e.g., using `perform_UMAP`). 957 - Cluster labels are added only if present in `self.clustering_metadata`. 958 """ 959 960 umap_data = self.umap 961 962 try: 963 umap_data["clusters"] = self.clustering_metadata["UMAP_clusters"] 964 except: 965 pass 966 967 return umap_data 968 969 def get_pca_data(self): 970 """ 971 Retrieve PCA embedding data with optional cluster labels. 972 973 Returns the principal component scores stored in `self.pca`. If clustering 974 metadata is available (specifically `PCA_clusters`), the corresponding 975 cluster assignments are appended as an additional column. 976 977 Returns 978 ------- 979 pandas.DataFrame 980 DataFrame containing PCA coordinates (columns: 'PC1', 'PC2', ...). 981 If available, includes an extra column 'clusters' with cluster labels. 982 983 Notes 984 ----- 985 - PCA must be computed beforehand (e.g., using `perform_PCA`). 986 - Cluster labels are added only if present in `self.clustering_metadata`. 987 """ 988 989 pca_data = self.pca 990 991 try: 992 pca_data["clusters"] = self.clustering_metadata["PCA_clusters"] 993 except: 994 pass 995 996 return pca_data 997 998 def return_clusters(self, clusters="umap"): 999 """ 1000 Retrieve cluster labels from UMAP or PCA clustering results. 1001 1002 Parameters 1003 ---------- 1004 clusters : str, default 'umap' 1005 Source of cluster labels to return. Must be one of: 1006 - 'umap': return cluster labels from UMAP embeddings. 1007 - 'pca' : return cluster labels from PCA embeddings. 1008 1009 Returns 1010 ------- 1011 list 1012 Cluster labels corresponding to the selected embedding method. 1013 1014 Raises 1015 ------ 1016 ValueError 1017 If `clusters` is not 'umap' or 'pca'. 1018 1019 Notes 1020 ----- 1021 Requires that clustering has already been performed 1022 (e.g., using `find_clusters_UMAP` or `find_clusters_PCA`). 1023 """ 1024 1025 if clusters.lower() == "umap": 1026 clusters_vector = self.clustering_metadata["UMAP_clusters"] 1027 elif clusters.lower() == "pca": 1028 clusters_vector = self.clustering_metadata["PCA_clusters"] 1029 else: 1030 raise ValueError("Parameter 'clusters' must be either 'umap' or 'pca'.") 1031 1032 return clusters_vector
A class for performing dimensionality reduction, clustering, and visualization on high-dimensional data (e.g., single-cell gene expression).
The class provides methods for:
- Normalizing and extracting subsets of data
- Principal Component Analysis (PCA) and related clustering
- Uniform Manifold Approximation and Projection (UMAP) and clustering
- Visualization of PCA and UMAP embeddings
- Harmonization of batch effects
- Accessing processed data and cluster labels
Methods
add_data_frame(data, metadata) Class method to create a Clustering instance from a DataFrame and metadata.
harmonize_sets() Perform batch effect harmonization on PCA data.
perform_PCA(pc_num=100, width=8, height=6) Perform PCA on the dataset and visualize the first two PCs.
knee_plot_PCA(width=8, height=6) Plot the cumulative variance explained by PCs to determine optimal dimensionality.
find_clusters_PCA(pc_num=0, eps=0.5, min_samples=10, width=8, height=6, harmonized=False) Apply DBSCAN clustering to PCA embeddings and visualize results.
perform_UMAP(factorize=False, umap_num=100, pc_num=0, harmonized=False, ...) Compute UMAP embeddings with optional parameter tuning.
knee_plot_umap(eps=0.5, min_samples=10) Determine optimal UMAP dimensionality using silhouette scores.
find_clusters_UMAP(umap_n=5, eps=0.5, min_samples=10, width=8, height=6) Apply DBSCAN clustering on UMAP embeddings and visualize clusters.
UMAP_vis(names_slot='cell_names', set_sep=True, point_size=0.6, ...) Visualize UMAP embeddings with labels and optional cluster numbering.
UMAP_feature(feature_name, features_data=None, point_size=0.6, ...) Plot a single feature over UMAP coordinates with customizable colormap.
get_umap_data() Return the UMAP embeddings along with cluster labels if available.
get_pca_data() Return the PCA results along with cluster labels if available.
return_clusters(clusters='umap') Return the cluster labels for UMAP or PCA embeddings.
Raises
ValueError For invalid parameters, mismatched dimensions, or missing metadata.
Clustering(data, metadata)
93 def __init__(self, data, metadata): 94 """ 95 Initialize the clustering class with data and optional metadata. 96 97 Parameters 98 ---------- 99 data : pandas.DataFrame 100 Input data for clustering. Columns are considered as samples. 101 102 metadata : pandas.DataFrame, optional 103 Metadata for the samples. If None, a default DataFrame with column 104 names as 'cell_names' is created. 105 106 Attributes 107 ---------- 108 -clustering_data : pandas.DataFrame 109 -clustering_metadata : pandas.DataFrame 110 -subclusters : None or dict 111 -explained_var : None or numpy.ndarray 112 -cumulative_var : None or numpy.ndarray 113 -pca : None or pandas.DataFrame 114 -harmonized_pca : None or pandas.DataFrame 115 -umap : None or pandas.DataFrame 116 """ 117 118 self.clustering_data = data 119 """The input data used for clustering.""" 120 121 if metadata is None: 122 metadata = pd.DataFrame({"cell_names": list(data.columns)}) 123 124 self.clustering_metadata = metadata 125 """Metadata associated with the samples.""" 126 127 self.subclusters = None 128 """Placeholder for storing subcluster information.""" 129 130 self.explained_var = None 131 """Explained variance from PCA, initialized as None.""" 132 133 self.cumulative_var = None 134 """Cumulative explained variance from PCA, initialized as None.""" 135 136 self.pca = None 137 """PCA-transformed data, initialized as None.""" 138 139 self.harmonized_pca = None 140 """PCA data after batch effect harmonization, initialized as None.""" 141 142 self.umap = None 143 """UMAP embeddings, initialized as None."""
Initialize the clustering class with data and optional metadata.
Parameters
data : pandas.DataFrame Input data for clustering. Columns are considered as samples.
metadata : pandas.DataFrame, optional Metadata for the samples. If None, a default DataFrame with column names as 'cell_names' is created.
Attributes
-clustering_data : pandas.DataFrame -clustering_metadata : pandas.DataFrame -subclusters : None or dict -explained_var : None or numpy.ndarray -cumulative_var : None or numpy.ndarray -pca : None or pandas.DataFrame -harmonized_pca : None or pandas.DataFrame -umap : None or pandas.DataFrame
@classmethod
def
add_data_frame( cls, data: pandas.core.frame.DataFrame, metadata: pandas.core.frame.DataFrame | None):
145 @classmethod 146 def add_data_frame(cls, data: pd.DataFrame, metadata: pd.DataFrame | None): 147 """ 148 Create a Clustering instance from a DataFrame and optional metadata. 149 150 Parameters 151 ---------- 152 data : pandas.DataFrame 153 Input data with features as rows and samples/cells as columns. 154 155 metadata : pandas.DataFrame or None 156 Optional metadata for the samples. 157 Each row corresponds to a sample/cell, and column names in this DataFrame 158 should match the sample/cell names in `data`. Columns can contain additional 159 information such as cell type, experimental condition, batch, sets, etc. 160 161 Returns 162 ------- 163 Clustering 164 A new instance of the Clustering class. 165 """ 166 167 return cls(data, metadata)
Create a Clustering instance from a DataFrame and optional metadata.
Parameters
data : pandas.DataFrame Input data with features as rows and samples/cells as columns.
metadata : pandas.DataFrame or None
Optional metadata for the samples.
Each row corresponds to a sample/cell, and column names in this DataFrame
should match the sample/cell names in data. Columns can contain additional
information such as cell type, experimental condition, batch, sets, etc.
Returns
Clustering A new instance of the Clustering class.
def
harmonize_sets(self, batch_col: str = 'sets'):
169 def harmonize_sets(self, batch_col: str = "sets"): 170 """ 171 Perform batch effect harmonization on PCA embeddings. 172 173 Parameters 174 ---------- 175 batch_col : str, default 'sets' 176 Name of the column in `metadata` that contains batch information for the samples/cells. 177 178 Returns 179 ------- 180 None 181 Updates the `harmonized_pca` attribute with harmonized data. 182 """ 183 184 data_mat = np.array(self.pca) 185 186 metadata = self.clustering_metadata 187 188 self.harmonized_pca = pd.DataFrame( 189 harmonize.run_harmony(data_mat, metadata, vars_use=batch_col).Z_corr 190 ).T 191 192 self.harmonized_pca.columns = self.pca.columns
Perform batch effect harmonization on PCA embeddings.
Parameters
batch_col : str, default 'sets'
Name of the column in metadata that contains batch information for the samples/cells.
Returns
None
Updates the harmonized_pca attribute with harmonized data.
def
perform_PCA(self, pc_num: int = 100, width=8, height=6):
194 def perform_PCA(self, pc_num: int = 100, width=8, height=6): 195 """ 196 Perform Principal Component Analysis (PCA) on the dataset. 197 198 This method standardizes the data, applies PCA, stores results as attributes, 199 and generates a scatter plot of the first two principal components. 200 201 Parameters 202 ---------- 203 pc_num : int, default 100 204 Number of principal components to compute. 205 If 0, computes all available components. 206 207 width : int or float, default 8 208 Width of the PCA figure. 209 210 height : int or float, default 6 211 Height of the PCA figure. 212 213 Returns 214 ------- 215 matplotlib.figure.Figure 216 Scatter plot showing the first two principal components. 217 218 Updates 219 ------- 220 self.pca : pandas.DataFrame 221 DataFrame with principal component scores for each sample. 222 223 self.explained_var : numpy.ndarray 224 Percentage of variance explained by each principal component. 225 226 self.cumulative_var : numpy.ndarray 227 Cumulative explained variance. 228 """ 229 230 scaler = StandardScaler() 231 data_scaled = scaler.fit_transform(self.clustering_data.T) 232 233 if pc_num == 0 or pc_num > data_scaled.shape[0]: 234 pc_num = data_scaled.shape[0] 235 236 pca = PCA(n_components=pc_num, random_state=42) 237 238 principal_components = pca.fit_transform(data_scaled) 239 240 pca_df = pd.DataFrame( 241 data=principal_components, 242 columns=["PC" + str(x + 1) for x in range(pc_num)], 243 ) 244 245 self.explained_var = pca.explained_variance_ratio_ * 100 246 self.cumulative_var = np.cumsum(self.explained_var) 247 248 self.pca = pca_df 249 250 fig = plt.figure(figsize=(width, height)) 251 plt.scatter(pca_df["PC1"], pca_df["PC2"], alpha=0.7) 252 plt.xlabel("PC 1") 253 plt.ylabel("PC 2") 254 plt.grid(True) 255 plt.show() 256 257 return fig
Perform Principal Component Analysis (PCA) on the dataset.
This method standardizes the data, applies PCA, stores results as attributes, and generates a scatter plot of the first two principal components.
Parameters
pc_num : int, default 100 Number of principal components to compute. If 0, computes all available components.
width : int or float, default 8 Width of the PCA figure.
height : int or float, default 6 Height of the PCA figure.
Returns
matplotlib.figure.Figure Scatter plot showing the first two principal components.
Updates
self.pca : pandas.DataFrame DataFrame with principal component scores for each sample.
self.explained_var : numpy.ndarray Percentage of variance explained by each principal component.
self.cumulative_var : numpy.ndarray Cumulative explained variance.
def
knee_plot_PCA(self, width: int = 8, height: int = 6):
259 def knee_plot_PCA(self, width: int = 8, height: int = 6): 260 """ 261 Plot cumulative explained variance to determine the optimal number of PCs. 262 263 Parameters 264 ---------- 265 width : int, default 8 266 Width of the figure. 267 268 height : int or, default 6 269 Height of the figure. 270 271 Returns 272 ------- 273 matplotlib.figure.Figure 274 Line plot showing cumulative variance explained by each PC. 275 """ 276 277 fig_knee = plt.figure(figsize=(width, height)) 278 plt.plot(range(1, len(self.explained_var) + 1), self.cumulative_var, marker="o") 279 plt.xlabel("PC (n components)") 280 plt.ylabel("Cumulative explained variance (%)") 281 plt.grid(True) 282 283 xticks = [1] + list(range(5, len(self.explained_var) + 1, 5)) 284 plt.xticks(xticks, rotation=60) 285 286 plt.show() 287 288 return fig_knee
Plot cumulative explained variance to determine the optimal number of PCs.
Parameters
width : int, default 8 Width of the figure.
height : int or, default 6 Height of the figure.
Returns
matplotlib.figure.Figure Line plot showing cumulative variance explained by each PC.
def
find_clusters_PCA( self, pc_num: int = 2, eps: float = 0.5, min_samples: int = 10, width: int = 8, height: int = 6, harmonized: bool = False):
290 def find_clusters_PCA( 291 self, 292 pc_num: int = 2, 293 eps: float = 0.5, 294 min_samples: int = 10, 295 width: int = 8, 296 height: int = 6, 297 harmonized: bool = False, 298 ): 299 """ 300 Apply DBSCAN clustering to PCA embeddings and visualize the results. 301 302 This method performs density-based clustering (DBSCAN) on the PCA-reduced 303 dataset. Cluster labels are stored in the object's metadata, and a scatter 304 plot of the first two principal components with cluster annotations is returned. 305 306 Parameters 307 ---------- 308 pc_num : int, default 2 309 Number of principal components to use for clustering. 310 If 0, uses all available components. 311 312 eps : float, default 0.5 313 Maximum distance between two points for them to be considered 314 as neighbors (DBSCAN parameter). 315 316 min_samples : int, default 10 317 Minimum number of samples required to form a cluster (DBSCAN parameter). 318 319 width : int, default 8 320 Width of the output scatter plot. 321 322 height : int, default 6 323 Height of the output scatter plot. 324 325 harmonized : bool, default False 326 If True, use harmonized PCA data (`self.harmonized_pca`). 327 If False, use standard PCA results (`self.pca`). 328 329 Returns 330 ------- 331 matplotlib.figure.Figure 332 Scatter plot of the first two principal components colored by 333 cluster assignments. 334 335 Updates 336 ------- 337 self.clustering_metadata['PCA_clusters'] : list 338 Cluster labels assigned to each cell/sample. 339 340 self.input_metadata['PCA_clusters'] : list, optional 341 Cluster labels stored in input metadata (if available). 342 """ 343 344 dbscan = DBSCAN(eps=eps, min_samples=min_samples) 345 346 if pc_num == 0 and harmonized: 347 PCA = self.harmonized_pca 348 349 elif pc_num == 0: 350 PCA = self.pca 351 352 else: 353 if harmonized: 354 355 PCA = self.harmonized_pca.iloc[:, 0:pc_num] 356 else: 357 358 PCA = self.pca.iloc[:, 0:pc_num] 359 360 dbscan_labels = dbscan.fit_predict(PCA) 361 362 pca_df = pd.DataFrame(PCA) 363 pca_df["Cluster"] = dbscan_labels 364 365 fig = plt.figure(figsize=(width, height)) 366 367 for cluster_id in sorted(pca_df["Cluster"].unique()): 368 cluster_data = pca_df[pca_df["Cluster"] == cluster_id] 369 plt.scatter( 370 cluster_data["PC1"], 371 cluster_data["PC2"], 372 label=f"Cluster {cluster_id}", 373 alpha=0.7, 374 ) 375 376 plt.xlabel("PC 1") 377 plt.ylabel("PC 2") 378 plt.legend(title="Clusters", loc="center left", bbox_to_anchor=(1.0, 0.5)) 379 380 plt.grid(True) 381 plt.show() 382 383 self.clustering_metadata["PCA_clusters"] = [str(x) for x in dbscan_labels] 384 385 try: 386 self.input_metadata["PCA_clusters"] = [str(x) for x in dbscan_labels] 387 except: 388 pass 389 390 return fig
Apply DBSCAN clustering to PCA embeddings and visualize the results.
This method performs density-based clustering (DBSCAN) on the PCA-reduced dataset. Cluster labels are stored in the object's metadata, and a scatter plot of the first two principal components with cluster annotations is returned.
Parameters
pc_num : int, default 2 Number of principal components to use for clustering. If 0, uses all available components.
eps : float, default 0.5 Maximum distance between two points for them to be considered as neighbors (DBSCAN parameter).
min_samples : int, default 10 Minimum number of samples required to form a cluster (DBSCAN parameter).
width : int, default 8 Width of the output scatter plot.
height : int, default 6 Height of the output scatter plot.
harmonized : bool, default False
If True, use harmonized PCA data (self.harmonized_pca).
If False, use standard PCA results (self.pca).
Returns
matplotlib.figure.Figure Scatter plot of the first two principal components colored by cluster assignments.
Updates
self.clustering_metadata['PCA_clusters'] : list Cluster labels assigned to each cell/sample.
self.input_metadata['PCA_clusters'] : list, optional Cluster labels stored in input metadata (if available).
def
perform_UMAP( self, factorize: bool = False, umap_num: int = 100, pc_num: int = 0, harmonized: bool = False, n_neighbors: int = 5, min_dist: float | int = 0.1, spread: float | int = 1.0, set_op_mix_ratio: float | int = 1.0, local_connectivity: int = 1, repulsion_strength: float | int = 1.0, negative_sample_rate: int = 5, width: int = 8, height: int = 6):
392 def perform_UMAP( 393 self, 394 factorize: bool = False, 395 umap_num: int = 100, 396 pc_num: int = 0, 397 harmonized: bool = False, 398 n_neighbors: int = 5, 399 min_dist: float | int = 0.1, 400 spread: float | int = 1.0, 401 set_op_mix_ratio: float | int = 1.0, 402 local_connectivity: int = 1, 403 repulsion_strength: float | int = 1.0, 404 negative_sample_rate: int = 5, 405 width: int = 8, 406 height: int = 6, 407 ): 408 """ 409 Compute and visualize UMAP embeddings of the dataset. 410 411 This method applies Uniform Manifold Approximation and Projection (UMAP) 412 for dimensionality reduction on either raw, PCA, or harmonized PCA data. 413 Results are stored as a DataFrame (`self.umap`) and a scatter plot figure 414 (`self.UMAP_plot`). 415 416 Parameters 417 ---------- 418 factorize : bool, default False 419 If True, categorical sample labels (from column names) are factorized 420 and used as supervision in UMAP fitting. 421 422 umap_num : int, default 100 423 Number of UMAP dimensions to compute. If 0, matches the input dimension. 424 425 pc_num : int, default 0 426 Number of principal components to use as UMAP input. 427 If 0, use all available components or raw data. 428 429 harmonized : bool, default False 430 If True, use harmonized PCA embeddings (`self.harmonized_pca`). 431 If False, use standard PCA or raw scaled data. 432 433 n_neighbors : int, default 5 434 UMAP parameter controlling the size of the local neighborhood. 435 436 min_dist : float, default 0.1 437 UMAP parameter controlling minimum allowed distance between embedded points. 438 439 spread : int | float, default 1.0 440 Effective scale of embedded space (UMAP parameter). 441 442 set_op_mix_ratio : int | float, default 1.0 443 Interpolation parameter between union and intersection in fuzzy sets. 444 445 local_connectivity : int, default 1 446 Number of nearest neighbors assumed for each point. 447 448 repulsion_strength : int | float, default 1.0 449 Weighting applied to negative samples during optimization. 450 451 negative_sample_rate : int, default 5 452 Number of negative samples per positive sample in optimization. 453 454 width : int, default 8 455 Width of the output scatter plot. 456 457 height : int, default 6 458 Height of the output scatter plot. 459 460 Updates 461 ------- 462 self.umap : pandas.DataFrame 463 Table of UMAP embeddings with columns `UMAP1 ... UMAPn`. 464 465 Notes 466 ----- 467 For supervised UMAP (`factorize=True`), categorical codes from column 468 names of the dataset are used as labels. 469 """ 470 471 scaler = StandardScaler() 472 473 if pc_num == 0 and harmonized: 474 data_scaled = self.harmonized_pca 475 476 elif pc_num == 0: 477 data_scaled = scaler.fit_transform(self.clustering_data.T) 478 479 else: 480 if harmonized: 481 482 data_scaled = self.harmonized_pca.iloc[:, 0:pc_num] 483 else: 484 485 data_scaled = self.pca.iloc[:, 0:pc_num] 486 487 if umap_num == 0 or umap_num > data_scaled.shape[1]: 488 489 umap_num = data_scaled.shape[1] 490 491 reducer = umap.UMAP( 492 n_components=len(data_scaled.T), 493 random_state=42, 494 n_neighbors=n_neighbors, 495 min_dist=min_dist, 496 spread=spread, 497 set_op_mix_ratio=set_op_mix_ratio, 498 local_connectivity=local_connectivity, 499 repulsion_strength=repulsion_strength, 500 negative_sample_rate=negative_sample_rate, 501 n_jobs=1, 502 ) 503 504 else: 505 506 reducer = umap.UMAP( 507 n_components=umap_num, 508 random_state=42, 509 n_neighbors=n_neighbors, 510 min_dist=min_dist, 511 spread=spread, 512 set_op_mix_ratio=set_op_mix_ratio, 513 local_connectivity=local_connectivity, 514 repulsion_strength=repulsion_strength, 515 negative_sample_rate=negative_sample_rate, 516 n_jobs=1, 517 ) 518 519 if factorize: 520 embedding = reducer.fit_transform( 521 X=data_scaled, y=pd.Categorical(self.clustering_data.columns).codes 522 ) 523 else: 524 embedding = reducer.fit_transform(X=data_scaled) 525 526 umap_df = pd.DataFrame( 527 embedding, columns=["UMAP" + str(x + 1) for x in range(umap_num)] 528 ) 529 530 plt.figure(figsize=(width, height)) 531 plt.scatter(umap_df["UMAP1"], umap_df["UMAP2"], alpha=0.7) 532 plt.xlabel("UMAP 1") 533 plt.ylabel("UMAP 2") 534 plt.grid(True) 535 536 plt.show() 537 538 self.umap = umap_df
Compute and visualize UMAP embeddings of the dataset.
This method applies Uniform Manifold Approximation and Projection (UMAP)
for dimensionality reduction on either raw, PCA, or harmonized PCA data.
Results are stored as a DataFrame (self.umap) and a scatter plot figure
(self.UMAP_plot).
Parameters
factorize : bool, default False If True, categorical sample labels (from column names) are factorized and used as supervision in UMAP fitting.
umap_num : int, default 100 Number of UMAP dimensions to compute. If 0, matches the input dimension.
pc_num : int, default 0 Number of principal components to use as UMAP input. If 0, use all available components or raw data.
harmonized : bool, default False
If True, use harmonized PCA embeddings (self.harmonized_pca).
If False, use standard PCA or raw scaled data.
n_neighbors : int, default 5 UMAP parameter controlling the size of the local neighborhood.
min_dist : float, default 0.1 UMAP parameter controlling minimum allowed distance between embedded points.
spread : int | float, default 1.0 Effective scale of embedded space (UMAP parameter).
set_op_mix_ratio : int | float, default 1.0 Interpolation parameter between union and intersection in fuzzy sets.
local_connectivity : int, default 1 Number of nearest neighbors assumed for each point.
repulsion_strength : int | float, default 1.0 Weighting applied to negative samples during optimization.
negative_sample_rate : int, default 5 Number of negative samples per positive sample in optimization.
width : int, default 8 Width of the output scatter plot.
height : int, default 6 Height of the output scatter plot.
Updates
self.umap : pandas.DataFrame
Table of UMAP embeddings with columns UMAP1 ... UMAPn.
Notes
For supervised UMAP (factorize=True), categorical codes from column
names of the dataset are used as labels.
def
knee_plot_umap(self, eps: int | float = 0.5, min_samples: int = 10):
540 def knee_plot_umap(self, eps: int | float = 0.5, min_samples: int = 10): 541 """ 542 Plot silhouette scores for different UMAP dimensions to determine optimal n_components. 543 544 Parameters 545 ---------- 546 eps : float, default 0.5 547 DBSCAN eps parameter for clustering each UMAP dimension. 548 549 min_samples : int, default 10 550 Minimum number of samples to form a cluster in DBSCAN. 551 552 Returns 553 ------- 554 matplotlib.figure.Figure 555 Silhouette score plot across UMAP dimensions. 556 """ 557 558 umap_range = range(2, len(self.umap.T) + 1) 559 560 silhouette_scores = [] 561 component = [] 562 for n in umap_range: 563 564 db = DBSCAN(eps=eps, min_samples=min_samples) 565 labels = db.fit_predict(np.array(self.umap)[:, :n]) 566 567 mask = labels != -1 568 if len(set(labels[mask])) > 1: 569 score = silhouette_score(np.array(self.umap)[:, :n][mask], labels[mask]) 570 else: 571 score = -1 572 573 silhouette_scores.append(score) 574 component.append(n) 575 576 fig = plt.figure(figsize=(10, 5)) 577 plt.plot(component, silhouette_scores, marker="o") 578 plt.xlabel("UMAP (n_components)") 579 plt.ylabel("Silhouette Score") 580 plt.grid(True) 581 plt.xticks(range(int(min(component)), int(max(component)) + 1, 1)) 582 583 plt.show() 584 585 return fig
Plot silhouette scores for different UMAP dimensions to determine optimal n_components.
Parameters
eps : float, default 0.5 DBSCAN eps parameter for clustering each UMAP dimension.
min_samples : int, default 10 Minimum number of samples to form a cluster in DBSCAN.
Returns
matplotlib.figure.Figure Silhouette score plot across UMAP dimensions.
def
find_clusters_UMAP( self, umap_n: int = 5, eps: float = 0.5, min_samples: int = 10, width: int = 8, height: int = 6):
587 def find_clusters_UMAP( 588 self, 589 umap_n: int = 5, 590 eps: float | float = 0.5, 591 min_samples: int = 10, 592 width: int = 8, 593 height: int = 6, 594 ): 595 """ 596 Apply DBSCAN clustering on UMAP embeddings and visualize clusters. 597 598 This method performs density-based clustering (DBSCAN) on the UMAP-reduced 599 dataset. Cluster labels are stored in the object's metadata, and a scatter 600 plot of the first two UMAP components with cluster annotations is returned. 601 602 Parameters 603 ---------- 604 umap_n : int, default 5 605 Number of UMAP dimensions to use for DBSCAN clustering. 606 Must be <= number of columns in `self.umap`. 607 608 eps : float | int, default 0.5 609 Maximum neighborhood distance between two samples for them to be considered 610 as in the same cluster (DBSCAN parameter). 611 612 min_samples : int, default 10 613 Minimum number of samples in a neighborhood to form a cluster (DBSCAN parameter). 614 615 width : int, default 8 616 Figure width. 617 618 height : int, default 6 619 Figure height. 620 621 Returns 622 ------- 623 matplotlib.figure.Figure 624 Scatter plot of the first two UMAP components colored by 625 cluster assignments. 626 627 Updates 628 ------- 629 self.clustering_metadata['UMAP_clusters'] : list 630 Cluster labels assigned to each cell/sample. 631 632 self.input_metadata['UMAP_clusters'] : list, optional 633 Cluster labels stored in input metadata (if available). 634 """ 635 636 dbscan = DBSCAN(eps=eps, min_samples=min_samples) 637 dbscan_labels = dbscan.fit_predict(np.array(self.umap)[:, :umap_n]) 638 639 umap_df = self.umap 640 umap_df["Cluster"] = dbscan_labels 641 642 fig = plt.figure(figsize=(width, height)) 643 644 for cluster_id in sorted(umap_df["Cluster"].unique()): 645 cluster_data = umap_df[umap_df["Cluster"] == cluster_id] 646 plt.scatter( 647 cluster_data["UMAP1"], 648 cluster_data["UMAP2"], 649 label=f"Cluster {cluster_id}", 650 alpha=0.7, 651 ) 652 653 plt.xlabel("UMAP 1") 654 plt.ylabel("UMAP 2") 655 plt.legend(title="Clusters", loc="center left", bbox_to_anchor=(1.0, 0.5)) 656 plt.grid(True) 657 658 self.clustering_metadata["UMAP_clusters"] = [str(x) for x in dbscan_labels] 659 660 try: 661 self.input_metadata["UMAP_clusters"] = [str(x) for x in dbscan_labels] 662 except: 663 pass 664 665 return fig
Apply DBSCAN clustering on UMAP embeddings and visualize clusters.
This method performs density-based clustering (DBSCAN) on the UMAP-reduced dataset. Cluster labels are stored in the object's metadata, and a scatter plot of the first two UMAP components with cluster annotations is returned.
Parameters
umap_n : int, default 5
Number of UMAP dimensions to use for DBSCAN clustering.
Must be <= number of columns in self.umap.
eps : float | int, default 0.5 Maximum neighborhood distance between two samples for them to be considered as in the same cluster (DBSCAN parameter).
min_samples : int, default 10 Minimum number of samples in a neighborhood to form a cluster (DBSCAN parameter).
width : int, default 8 Figure width.
height : int, default 6 Figure height.
Returns
matplotlib.figure.Figure Scatter plot of the first two UMAP components colored by cluster assignments.
Updates
self.clustering_metadata['UMAP_clusters'] : list Cluster labels assigned to each cell/sample.
self.input_metadata['UMAP_clusters'] : list, optional Cluster labels stored in input metadata (if available).
def
UMAP_vis( self, names_slot: str = 'cell_names', set_sep: bool = True, point_size: int | float = 0.6, font_size: int | float = 6, legend_split_col: int = 2, width: int = 8, height: int = 6, inc_num: bool = True):
667 def UMAP_vis( 668 self, 669 names_slot: str = "cell_names", 670 set_sep: bool = True, 671 point_size: int | float = 0.6, 672 font_size: int | float = 6, 673 legend_split_col: int = 2, 674 width: int = 8, 675 height: int = 6, 676 inc_num: bool = True, 677 ): 678 """ 679 Visualize UMAP embeddings with sample labels based on specyfic metadata slot. 680 681 Parameters 682 ---------- 683 names_slot : str, default 'cell_names' 684 Column in metadata to use as sample labels. 685 686 set_sep : bool, default True 687 If True, separate points by dataset. 688 689 point_size : float, default 0.6 690 Size of scatter points. 691 692 font_size : int, default 6 693 Font size for numbers on points. 694 695 legend_split_col : int, default 2 696 Number of columns in legend. 697 698 width : int, default 8 699 Figure width. 700 701 height : int, default 6 702 Figure height. 703 704 inc_num : bool, default True 705 If True, annotate points with numeric labels. 706 707 Returns 708 ------- 709 matplotlib.figure.Figure 710 UMAP scatter plot figure. 711 """ 712 713 umap_df = self.umap.iloc[:, 0:2].copy() 714 umap_df["names"] = list(self.clustering_metadata[names_slot]) 715 716 if set_sep: 717 718 if "sets" in list(self.clustering_metadata.columns): 719 umap_df["dataset"] = list(self.clustering_metadata["sets"]) 720 else: 721 umap_df["dataset"] = "default" 722 723 else: 724 umap_df["dataset"] = "default" 725 726 umap_df["tmp_nam"] = list(umap_df["names"] + umap_df["dataset"]) 727 728 umap_df["count"] = umap_df["tmp_nam"].map(umap_df["tmp_nam"].value_counts()) 729 730 numeric_df = ( 731 pd.DataFrame(umap_df[["count", "tmp_nam", "names"]].copy()) 732 .drop_duplicates() 733 .sort_values("count", ascending=False) 734 ) 735 numeric_df["numeric_values"] = range(0, numeric_df.shape[0]) 736 737 umap_df = umap_df.merge( 738 numeric_df[["tmp_nam", "numeric_values"]], on="tmp_nam", how="left" 739 ) 740 741 fig, ax = plt.subplots(figsize=(width, height)) 742 743 markers = ["o", "s", "^", "D", "P", "*", "X"] 744 marker_map = { 745 ds: markers[i % len(markers)] 746 for i, ds in enumerate(umap_df["dataset"].unique()) 747 } 748 749 cord_list = [] 750 751 for num, nam in zip(numeric_df["numeric_values"], numeric_df["names"]): 752 753 cluster_data = umap_df[umap_df["numeric_values"] == num] 754 755 ax.scatter( 756 cluster_data["UMAP1"], 757 cluster_data["UMAP2"], 758 label=f"{num} - {nam}", 759 marker=marker_map[cluster_data["dataset"].iloc[0]], 760 alpha=0.6, 761 s=point_size, 762 ) 763 764 coords = cluster_data[["UMAP1", "UMAP2"]].values 765 766 dists = pairwise_distances(coords) 767 768 sum_dists = dists.sum(axis=1) 769 770 center_idx = np.argmin(sum_dists) 771 center_point = coords[center_idx] 772 773 cord_list.append(center_point) 774 775 if inc_num: 776 texts = [] 777 for (x, y), num in zip(cord_list, numeric_df["numeric_values"]): 778 texts.append( 779 ax.text( 780 x, 781 y, 782 str(num), 783 ha="center", 784 va="center", 785 fontsize=font_size, 786 color="black", 787 ) 788 ) 789 790 adjust_text(texts, arrowprops=dict(arrowstyle="-", color="gray", lw=0.5)) 791 792 ax.set_xlabel("UMAP 1") 793 ax.set_ylabel("UMAP 2") 794 795 ax.legend( 796 title="Clusters", 797 loc="center left", 798 bbox_to_anchor=(1.05, 0.5), 799 ncol=legend_split_col, 800 markerscale=5, 801 ) 802 803 ax.grid(True) 804 805 plt.tight_layout() 806 807 return fig
Visualize UMAP embeddings with sample labels based on specyfic metadata slot.
Parameters
names_slot : str, default 'cell_names' Column in metadata to use as sample labels.
set_sep : bool, default True If True, separate points by dataset.
point_size : float, default 0.6 Size of scatter points.
font_size : int, default 6 Font size for numbers on points.
legend_split_col : int, default 2 Number of columns in legend.
width : int, default 8 Figure width.
height : int, default 6 Figure height.
inc_num : bool, default True If True, annotate points with numeric labels.
Returns
matplotlib.figure.Figure UMAP scatter plot figure.
def
UMAP_feature( self, feature_name: str, features_data: pandas.core.frame.DataFrame | None, point_size: int | float = 0.6, font_size: int | float = 6, width: int = 8, height: int = 6, palette='light'):
809 def UMAP_feature( 810 self, 811 feature_name: str, 812 features_data: pd.DataFrame | None, 813 point_size: int | float = 0.6, 814 font_size: int | float = 6, 815 width: int = 8, 816 height: int = 6, 817 palette="light", 818 ): 819 """ 820 Visualize UMAP embedding with expression levels of a selected feature. 821 822 Each point (cell) in the UMAP plot is colored according to the expression 823 value of the chosen feature, enabling interpretation of spatial patterns 824 of gene activity or metadata distribution in low-dimensional space. 825 826 Parameters 827 ---------- 828 feature_name : str 829 Name of the feature to plot. 830 831 features_data : pandas.DataFrame or None, default None 832 If None, the function uses the DataFrame containing the clustering data. 833 To plot features not used in clustering, provide a wider DataFrame 834 containing the original feature values. 835 836 point_size : float, default 0.6 837 Size of scatter points in the plot. 838 839 font_size : int, default 6 840 Font size for axis labels and annotations. 841 842 width : int, default 8 843 Width of the matplotlib figure. 844 845 height : int, default 6 846 Height of the matplotlib figure. 847 848 palette : str, default 'light' 849 Color palette for expression visualization. Options are: 850 - 'light' 851 - 'dark' 852 - 'green' 853 - 'gray' 854 855 Returns 856 ------- 857 matplotlib.figure.Figure 858 UMAP scatter plot colored by feature values. 859 """ 860 861 umap_df = self.umap.iloc[:, 0:2].copy() 862 863 if features_data is None: 864 865 features_data = self.clustering_data 866 867 if features_data.shape[1] != umap_df.shape[0]: 868 raise ValueError( 869 "Imputed 'features_data' shape does not match the number of UMAP cells" 870 ) 871 872 blist = [ 873 True if x.upper() == feature_name.upper() else False 874 for x in features_data.index 875 ] 876 877 if not any(blist): 878 raise ValueError("Imputed feature_name is not included in the data") 879 880 umap_df.loc[:, "feature"] = ( 881 features_data.loc[blist, :] 882 .apply(lambda row: row.tolist(), axis=1) 883 .values[0] 884 ) 885 886 umap_df = umap_df.sort_values("feature", ascending=True) 887 888 import matplotlib.colors as mcolors 889 890 if palette == "light": 891 palette = px.colors.sequential.Sunsetdark 892 893 elif palette == "dark": 894 palette = px.colors.sequential.thermal 895 896 elif palette == "green": 897 palette = px.colors.sequential.Aggrnyl 898 899 elif palette == "gray": 900 palette = px.colors.sequential.gray 901 palette = palette[::-1] 902 903 else: 904 raise ValueError( 905 'Palette not found. Use: "light", "dark", "gray", or "green"' 906 ) 907 908 converted = [] 909 for c in palette: 910 rgb_255 = px.colors.unlabel_rgb(c) 911 rgb_01 = tuple(v / 255.0 for v in rgb_255) 912 converted.append(rgb_01) 913 914 my_cmap = mcolors.ListedColormap(converted, name="custom") 915 916 fig, ax = plt.subplots(figsize=(width, height)) 917 sc = ax.scatter( 918 umap_df["UMAP1"], 919 umap_df["UMAP2"], 920 c=umap_df["feature"], 921 s=point_size, 922 cmap=my_cmap, 923 alpha=1.0, 924 edgecolors="black", 925 linewidths=0.1, 926 ) 927 928 cbar = plt.colorbar(sc, ax=ax) 929 cbar.set_label(f"{feature_name}") 930 931 ax.set_xlabel("UMAP 1") 932 ax.set_ylabel("UMAP 2") 933 934 ax.grid(True) 935 936 plt.tight_layout() 937 938 return fig
Visualize UMAP embedding with expression levels of a selected feature.
Each point (cell) in the UMAP plot is colored according to the expression value of the chosen feature, enabling interpretation of spatial patterns of gene activity or metadata distribution in low-dimensional space.
Parameters
feature_name : str Name of the feature to plot.
features_data : pandas.DataFrame or None, default None If None, the function uses the DataFrame containing the clustering data. To plot features not used in clustering, provide a wider DataFrame containing the original feature values.
point_size : float, default 0.6 Size of scatter points in the plot.
font_size : int, default 6 Font size for axis labels and annotations.
width : int, default 8 Width of the matplotlib figure.
height : int, default 6 Height of the matplotlib figure.
palette : str, default 'light' Color palette for expression visualization. Options are: - 'light' - 'dark' - 'green' - 'gray'
Returns
matplotlib.figure.Figure UMAP scatter plot colored by feature values.
def
get_umap_data(self):
940 def get_umap_data(self): 941 """ 942 Retrieve UMAP embedding data with optional cluster labels. 943 944 Returns the UMAP coordinates stored in `self.umap`. If clustering 945 metadata is available (specifically `UMAP_clusters`), the corresponding 946 cluster assignments are appended as an additional column. 947 948 Returns 949 ------- 950 pandas.DataFrame 951 DataFrame containing UMAP coordinates (columns: 'UMAP1', 'UMAP2', ...). 952 If available, includes an extra column 'clusters' with cluster labels. 953 954 Notes 955 ----- 956 - UMAP embeddings must be computed beforehand (e.g., using `perform_UMAP`). 957 - Cluster labels are added only if present in `self.clustering_metadata`. 958 """ 959 960 umap_data = self.umap 961 962 try: 963 umap_data["clusters"] = self.clustering_metadata["UMAP_clusters"] 964 except: 965 pass 966 967 return umap_data
Retrieve UMAP embedding data with optional cluster labels.
Returns the UMAP coordinates stored in self.umap. If clustering
metadata is available (specifically UMAP_clusters), the corresponding
cluster assignments are appended as an additional column.
Returns
pandas.DataFrame DataFrame containing UMAP coordinates (columns: 'UMAP1', 'UMAP2', ...). If available, includes an extra column 'clusters' with cluster labels.
Notes
- UMAP embeddings must be computed beforehand (e.g., using
perform_UMAP). - Cluster labels are added only if present in
self.clustering_metadata.
def
get_pca_data(self):
969 def get_pca_data(self): 970 """ 971 Retrieve PCA embedding data with optional cluster labels. 972 973 Returns the principal component scores stored in `self.pca`. If clustering 974 metadata is available (specifically `PCA_clusters`), the corresponding 975 cluster assignments are appended as an additional column. 976 977 Returns 978 ------- 979 pandas.DataFrame 980 DataFrame containing PCA coordinates (columns: 'PC1', 'PC2', ...). 981 If available, includes an extra column 'clusters' with cluster labels. 982 983 Notes 984 ----- 985 - PCA must be computed beforehand (e.g., using `perform_PCA`). 986 - Cluster labels are added only if present in `self.clustering_metadata`. 987 """ 988 989 pca_data = self.pca 990 991 try: 992 pca_data["clusters"] = self.clustering_metadata["PCA_clusters"] 993 except: 994 pass 995 996 return pca_data
Retrieve PCA embedding data with optional cluster labels.
Returns the principal component scores stored in self.pca. If clustering
metadata is available (specifically PCA_clusters), the corresponding
cluster assignments are appended as an additional column.
Returns
pandas.DataFrame DataFrame containing PCA coordinates (columns: 'PC1', 'PC2', ...). If available, includes an extra column 'clusters' with cluster labels.
Notes
- PCA must be computed beforehand (e.g., using
perform_PCA). - Cluster labels are added only if present in
self.clustering_metadata.
def
return_clusters(self, clusters='umap'):
998 def return_clusters(self, clusters="umap"): 999 """ 1000 Retrieve cluster labels from UMAP or PCA clustering results. 1001 1002 Parameters 1003 ---------- 1004 clusters : str, default 'umap' 1005 Source of cluster labels to return. Must be one of: 1006 - 'umap': return cluster labels from UMAP embeddings. 1007 - 'pca' : return cluster labels from PCA embeddings. 1008 1009 Returns 1010 ------- 1011 list 1012 Cluster labels corresponding to the selected embedding method. 1013 1014 Raises 1015 ------ 1016 ValueError 1017 If `clusters` is not 'umap' or 'pca'. 1018 1019 Notes 1020 ----- 1021 Requires that clustering has already been performed 1022 (e.g., using `find_clusters_UMAP` or `find_clusters_PCA`). 1023 """ 1024 1025 if clusters.lower() == "umap": 1026 clusters_vector = self.clustering_metadata["UMAP_clusters"] 1027 elif clusters.lower() == "pca": 1028 clusters_vector = self.clustering_metadata["PCA_clusters"] 1029 else: 1030 raise ValueError("Parameter 'clusters' must be either 'umap' or 'pca'.") 1031 1032 return clusters_vector
Retrieve cluster labels from UMAP or PCA clustering results.
Parameters
clusters : str, default 'umap' Source of cluster labels to return. Must be one of: - 'umap': return cluster labels from UMAP embeddings. - 'pca' : return cluster labels from PCA embeddings.
Returns
list Cluster labels corresponding to the selected embedding method.
Raises
ValueError
If clusters is not 'umap' or 'pca'.
Notes
Requires that clustering has already been performed
(e.g., using find_clusters_UMAP or find_clusters_PCA).
1035class COMPsc(Clustering): 1036 """ 1037 A class `COMPsc` (Comparison of single-cell data) designed for the integration, 1038 analysis, and visualization of single-cell datasets. 1039 The class supports independent dataset integration, subclustering of existing clusters, 1040 marker detection, and multiple visualization strategies. 1041 1042 The COMPsc class provides methods for: 1043 1044 - Normalizing and filtering single-cell data 1045 - Loading and saving sparse 10x-style datasets 1046 - Computing differential expression and marker genes 1047 - Clustering and subclustering analysis 1048 - Visualizing similarity and spatial relationships 1049 - Aggregating data by cell and set annotations 1050 - Managing metadata and renaming labels 1051 - Plotting gene detection histograms and feature dot plots 1052 1053 Methods 1054 ------- 1055 project_dir(path_to_directory, project_list) 1056 Scans a directory to create a COMPsc instance mapping project names to their paths. 1057 1058 save_project(name, path=os.getcwd()) 1059 Saves the COMPsc object to a pickle file on disk. 1060 1061 load_project(path) 1062 Loads a previously saved COMPsc object from a pickle file. 1063 1064 reduce_cols(reg, inc_set=False) 1065 Removes columns from data tables where column names contain a specified name or partial substring. 1066 1067 reduce_rows(reg, inc_set=False) 1068 Removes rows from data tables where column names contain a specified feature (gene) name. 1069 1070 get_data(set_info=False) 1071 Returns normalized data with optional set annotations in column names. 1072 1073 get_metadata() 1074 Returns the stored input metadata. 1075 1076 get_partial_data(names=None, features=None, name_slot='cell_names') 1077 Return a subset of the data by sample names and/or features. 1078 1079 gene_calculation() 1080 Calculates and stores per-cell gene detection counts as a pandas Series. 1081 1082 gene_histograme(bins=100) 1083 Plots a histogram of genes detected per cell with an overlaid normal distribution. 1084 1085 gene_threshold(min_n=None, max_n=None) 1086 Filters cells based on minimum and/or maximum gene detection thresholds. 1087 1088 load_sparse_from_projects(normalized_data=False) 1089 Loads and concatenates sparse 10x-style datasets from project paths into count or normalized data. 1090 1091 rename_names(mapping, slot='cell_names') 1092 Renames entries in a specified metadata column using a provided mapping dictionary. 1093 1094 rename_subclusters(mapping) 1095 Renames subcluster labels using a provided mapping dictionary. 1096 1097 save_sparse(path_to_save=os.getcwd(), name_slot='cell_names', data_slot='normalized') 1098 Exports data as 10x-compatible sparse files (matrix.mtx, barcodes.tsv, genes.tsv). 1099 1100 normalize_data(normalize=True, normalize_factor=100000) 1101 Normalizes raw counts to counts-per-specified factor (e.g., CPM-like). 1102 1103 statistic(cells=None, sets=None, min_exp=0.01, min_pct=0.1, n_proc=10) 1104 Computes per-feature differential expression statistics (Mann-Whitney U) comparing target vs. rest groups. 1105 1106 calculate_difference_markers(min_exp=0, min_pct=0.25, n_proc=10, force=False) 1107 Computes and caches differential markers using the statistic method. 1108 1109 clustering_features(features_list=None, name_slot='cell_names', p_val=0.05, top_n=25, adj_mean=True, beta=0.4) 1110 Prepares clustering input by selecting marker features and optionally smoothing cell values. 1111 1112 average() 1113 Aggregates normalized data by averaging across (cell_name, set) pairs. 1114 1115 estimating_similarity(method='pearson', p_val=0.05, top_n=25) 1116 Computes pairwise correlation and Euclidean distance between aggregated samples. 1117 1118 cell_genes_distribution(cells_names, cells_sets=None, image_width=10, image_high=7) 1119 Compares the distributions of detected genes per cell across selected cell populations using overlapping histograms. 1120 1121 mm_cell_genes(cells_names, cells_sets=None, non_include_zeros=True, image_width=7, image_high=10) 1122 Calculates and visualizes the mean and median gene expression across selected cell populations as a grouped bar plot. 1123 1124 cell_dispersion(cells_names, cells_sets=None, image_width=7, image_high=10) 1125 Assesses within-cluster transcriptional dispersion by calculating distances between individual cells and their cluster centroids, visualized via violin plots. 1126 1127 similarity_plot(split_sets=True, set_info=True, cmap='seismic', width=12, height=10) 1128 Visualizes pairwise similarity as a dot plot with correlation as hue and scaled distance as point size. 1129 1130 spatial_similarity(set_info=True, bandwidth=1, n_neighbors=5, min_dist=0.1, legend_split=2, point_size=20, ...) 1131 Creates a UMAP-like visualization of similarity relationships with cluster hulls and nearest-neighbor arrows. 1132 1133 subcluster_prepare(features, cluster) 1134 Initializes a Clustering object for subcluster analysis on a selected parent cluster. 1135 1136 define_subclusters(umap_num=2, eps=0.5, min_samples=10, bandwidth=1, n_neighbors=5, min_dist=0.1, ...) 1137 Performs UMAP and DBSCAN clustering on prepared subcluster data and stores cluster labels. 1138 1139 subcluster_features_dot(colors='viridis', hclust='complete', img_width=3, img_high=5, label_size=6, ...) 1140 Visualizes averaged expression and occurrence of features for subclusters as a dot plot. 1141 1142 subcluster_DEG_dot(top_n=3, min_exp=0, min_pct=0.25, p_val=0.05, colors='viridis', ...) 1143 Plots top differential features for subclusters as a features-dot visualization. 1144 1145 accept_subclusters() 1146 Commits subcluster labels to main metadata by renaming cell names and clears subcluster data. 1147 1148 Raises 1149 ------ 1150 ValueError 1151 For invalid parameters, mismatched dimensions, or missing metadata. 1152 1153 """ 1154 1155 def __init__( 1156 self, 1157 objects=None, 1158 ): 1159 """ 1160 Initialize the COMPsc class for single-cell data integration and analysis. 1161 1162 Parameters 1163 ---------- 1164 objects : list or None, optional 1165 Optional list of data objects to initialize the instance with. 1166 1167 Attributes 1168 ---------- 1169 -objects : list or None 1170 -input_data : pandas.DataFrame or None 1171 -input_metadata : pandas.DataFrame or None 1172 -normalized_data : pandas.DataFrame or None 1173 -agg_metadata : pandas.DataFrame or None 1174 -agg_normalized_data : pandas.DataFrame or None 1175 -similarity : pandas.DataFrame or None 1176 -var_data : pandas.DataFrame or None 1177 -subclusters_ : instance of Clustering class or None 1178 -cells_calc : pandas.Series or None 1179 -gene_calc : pandas.Series or None 1180 -composition_data : pandas.DataFrame or None 1181 """ 1182 1183 self.objects = objects 1184 """ Stores the input data objects.""" 1185 1186 self.input_data = None 1187 """Raw input data for clustering or integration analysis.""" 1188 1189 self.input_metadata = None 1190 """Metadata associated with the input data.""" 1191 1192 self.normalized_data = None 1193 """Normalized version of the input data.""" 1194 1195 self.agg_metadata = None 1196 '''Aggregated metadata for all sets in object related to "agg_normalized_data"''' 1197 1198 self.agg_normalized_data = None 1199 """Aggregated and normalized data across multiple sets.""" 1200 1201 self.similarity = None 1202 """Similarity data between cells across all samples. and sets""" 1203 1204 self.var_data = None 1205 """DEG analysis results summarizing variance across all samples in the object.""" 1206 1207 self.subclusters_ = None 1208 """Placeholder for information about subclusters analysis; if computed.""" 1209 1210 self.cells_calc = None 1211 """Number of cells detected per sample (grouped by lineage, e.g., cluster or name), reflecting data composition.""" 1212 1213 self.gene_calc = None 1214 """Number of genes detected per sample (cell), reflecting the sequencing depth.""" 1215 1216 self.composition_data = None 1217 """Data describing composition of cells across clusters or sets.""" 1218 1219 @classmethod 1220 def project_dir(cls, path_to_directory, project_list): 1221 """ 1222 Scan a directory and build a COMPsc instance mapping provided project names 1223 to their paths. 1224 1225 Parameters 1226 ---------- 1227 path_to_directory : str 1228 Path containing project subfolders. 1229 1230 project_list : list[str] 1231 List of filenames (folder names) to include in the returned object map. 1232 1233 Returns 1234 ------- 1235 COMPsc 1236 New COMPsc instance with `objects` populated. 1237 1238 Raises 1239 ------ 1240 Exception 1241 A generic exception is caught and a message printed if scanning fails. 1242 1243 Notes 1244 ----- 1245 Function attempts to match entries in `project_list` to directory 1246 names and constructs a simplified object key from the folder name. 1247 """ 1248 try: 1249 objects = {} 1250 for filename in tqdm(os.listdir(path_to_directory)): 1251 for c in project_list: 1252 f = os.path.join(path_to_directory, filename) 1253 if c == filename and os.path.isdir(f): 1254 objects[str(c)] = f 1255 1256 return cls(objects) 1257 1258 except: 1259 print("Something went wrong. Check the function input data and try again!") 1260 1261 def save_project(self, name, path: str = os.getcwd()): 1262 """ 1263 Save the COMPsc object to disk using pickle. 1264 1265 Parameters 1266 ---------- 1267 name : str 1268 Base filename (without extension) to use when saving. 1269 1270 path : str, default os.getcwd() 1271 Directory in which to save the project file. 1272 1273 Returns 1274 ------- 1275 None 1276 1277 Side Effects 1278 ------------ 1279 - Writes a file `<path>/<name>.jpkl` containing the pickled object. 1280 - Prints a confirmation message with saved path. 1281 """ 1282 1283 full = os.path.join(path, f"{name}.jpkl") 1284 1285 with open(full, "wb") as f: 1286 pickle.dump(self, f) 1287 1288 print(f"Project saved as {full}") 1289 1290 @classmethod 1291 def load_project(cls, path): 1292 """ 1293 Load a previously saved COMPsc project from a pickle file. 1294 1295 Parameters 1296 ---------- 1297 path : str 1298 Full path to the pickled project file. 1299 1300 Returns 1301 ------- 1302 COMPsc 1303 The unpickled COMPsc object. 1304 1305 Raises 1306 ------ 1307 FileNotFoundError 1308 If the provided path does not exist. 1309 """ 1310 1311 if not os.path.exists(path): 1312 raise FileNotFoundError("File does not exist!") 1313 with open(path, "rb") as f: 1314 obj = pickle.load(f) 1315 return obj 1316 1317 def reduce_cols( 1318 self, 1319 reg: str | None = None, 1320 full: str | None = None, 1321 name_slot: str = "cell_names", 1322 inc_set: bool = False, 1323 ): 1324 """ 1325 Remove columns (cells) whose names contain a substring `reg` or 1326 full name `full` from available tables. 1327 1328 Parameters 1329 ---------- 1330 reg : str | None 1331 Substring to search for in column/cell names; matching columns will be removed. 1332 If not None, `full` must be None. 1333 1334 full : str | None 1335 Full name to search for in column/cell names; matching columns will be removed. 1336 If not None, `reg` must be None. 1337 1338 name_slot : str, default 'cell_names' 1339 Column in metadata to use as sample names. 1340 1341 inc_set : bool, default False 1342 If True, column names are interpreted as 'cell_name # set' when matching. 1343 1344 Update 1345 ------------ 1346 Mutates `self.input_data`, `self.normalized_data`, `self.input_metadata`, 1347 `self.agg_normalized_data`, and `self.agg_metadata` (if they exist), 1348 removing columns/rows that match `reg`. 1349 1350 Raises 1351 ------ 1352 Raises ValueError if nothing matches the reduction mask. 1353 """ 1354 1355 if reg is None and full is None: 1356 raise ValueError( 1357 "Both 'reg' and 'full' arguments not provided. Please provide at least one of them!" 1358 ) 1359 1360 if reg is not None and full is not None: 1361 raise ValueError( 1362 "Both 'reg' and 'full' arguments are provided. " 1363 "Please provide only one of them!\n" 1364 "'reg' is used when only part of the name must be detected.\n" 1365 "'full' is used if the full name must be detected." 1366 ) 1367 1368 if reg is not None: 1369 1370 if self.input_data is not None: 1371 1372 if inc_set: 1373 1374 self.input_data.columns = ( 1375 self.input_metadata[name_slot] 1376 + " # " 1377 + self.input_metadata["sets"] 1378 ) 1379 1380 else: 1381 1382 self.input_data.columns = self.input_metadata[name_slot] 1383 1384 mask = [reg.upper() not in x.upper() for x in self.input_data.columns] 1385 1386 if len([y for y in mask if y is False]) == 0: 1387 raise ValueError("Nothing found to reduce") 1388 1389 self.input_data = self.input_data.loc[:, mask] 1390 1391 if self.normalized_data is not None: 1392 1393 if inc_set: 1394 1395 self.normalized_data.columns = ( 1396 self.input_metadata[name_slot] 1397 + " # " 1398 + self.input_metadata["sets"] 1399 ) 1400 1401 else: 1402 1403 self.normalized_data.columns = self.input_metadata[name_slot] 1404 1405 mask = [ 1406 reg.upper() not in x.upper() for x in self.normalized_data.columns 1407 ] 1408 1409 if len([y for y in mask if y is False]) == 0: 1410 raise ValueError("Nothing found to reduce") 1411 1412 self.normalized_data = self.normalized_data.loc[:, mask] 1413 1414 if self.input_metadata is not None: 1415 1416 if inc_set: 1417 1418 self.input_metadata["drop"] = ( 1419 self.input_metadata[name_slot] 1420 + " # " 1421 + self.input_metadata["sets"] 1422 ) 1423 1424 else: 1425 1426 self.input_metadata["drop"] = self.input_metadata[name_slot] 1427 1428 mask = [ 1429 reg.upper() not in x.upper() for x in self.input_metadata["drop"] 1430 ] 1431 1432 if len([y for y in mask if y is False]) == 0: 1433 raise ValueError("Nothing found to reduce") 1434 1435 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 1436 drop=True 1437 ) 1438 1439 self.input_metadata = self.input_metadata.drop( 1440 columns=["drop"], errors="ignore" 1441 ) 1442 1443 if self.agg_normalized_data is not None: 1444 1445 if inc_set: 1446 1447 self.agg_normalized_data.columns = ( 1448 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1449 ) 1450 1451 else: 1452 1453 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1454 1455 mask = [ 1456 reg.upper() not in x.upper() 1457 for x in self.agg_normalized_data.columns 1458 ] 1459 1460 if len([y for y in mask if y is False]) == 0: 1461 raise ValueError("Nothing found to reduce") 1462 1463 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 1464 1465 if self.agg_metadata is not None: 1466 1467 if inc_set: 1468 1469 self.agg_metadata["drop"] = ( 1470 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1471 ) 1472 1473 else: 1474 1475 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1476 1477 mask = [reg.upper() not in x.upper() for x in self.agg_metadata["drop"]] 1478 1479 if len([y for y in mask if y is False]) == 0: 1480 raise ValueError("Nothing found to reduce") 1481 1482 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 1483 drop=True 1484 ) 1485 1486 self.agg_metadata = self.agg_metadata.drop( 1487 columns=["drop"], errors="ignore" 1488 ) 1489 1490 elif full is not None: 1491 1492 if self.input_data is not None: 1493 1494 if inc_set: 1495 1496 self.input_data.columns = ( 1497 self.input_metadata[name_slot] 1498 + " # " 1499 + self.input_metadata["sets"] 1500 ) 1501 1502 if "#" not in full: 1503 1504 self.input_data.columns = self.input_metadata[name_slot] 1505 1506 print( 1507 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1508 "Only the names will be compared, without considering the set information." 1509 ) 1510 1511 else: 1512 1513 self.input_data.columns = self.input_metadata[name_slot] 1514 1515 mask = [full.upper() != x.upper() for x in self.input_data.columns] 1516 1517 if len([y for y in mask if y is False]) == 0: 1518 raise ValueError("Nothing found to reduce") 1519 1520 self.input_data = self.input_data.loc[:, mask] 1521 1522 if self.normalized_data is not None: 1523 1524 if inc_set: 1525 1526 self.normalized_data.columns = ( 1527 self.input_metadata[name_slot] 1528 + " # " 1529 + self.input_metadata["sets"] 1530 ) 1531 1532 if "#" not in full: 1533 1534 self.normalized_data.columns = self.input_metadata[name_slot] 1535 1536 print( 1537 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1538 "Only the names will be compared, without considering the set information." 1539 ) 1540 1541 else: 1542 1543 self.normalized_data.columns = self.input_metadata[name_slot] 1544 1545 mask = [full.upper() != x.upper() for x in self.normalized_data.columns] 1546 1547 if len([y for y in mask if y is False]) == 0: 1548 raise ValueError("Nothing found to reduce") 1549 1550 self.normalized_data = self.normalized_data.loc[:, mask] 1551 1552 if self.input_metadata is not None: 1553 1554 if inc_set: 1555 1556 self.input_metadata["drop"] = ( 1557 self.input_metadata[name_slot] 1558 + " # " 1559 + self.input_metadata["sets"] 1560 ) 1561 1562 if "#" not in full: 1563 1564 self.input_metadata["drop"] = self.input_metadata[name_slot] 1565 1566 print( 1567 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1568 "Only the names will be compared, without considering the set information." 1569 ) 1570 1571 else: 1572 1573 self.input_metadata["drop"] = self.input_metadata[name_slot] 1574 1575 mask = [full.upper() != x.upper() for x in self.input_metadata["drop"]] 1576 1577 if len([y for y in mask if y is False]) == 0: 1578 raise ValueError("Nothing found to reduce") 1579 1580 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 1581 drop=True 1582 ) 1583 1584 self.input_metadata = self.input_metadata.drop( 1585 columns=["drop"], errors="ignore" 1586 ) 1587 1588 if self.agg_normalized_data is not None: 1589 1590 if inc_set: 1591 1592 self.agg_normalized_data.columns = ( 1593 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1594 ) 1595 1596 if "#" not in full: 1597 1598 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1599 1600 print( 1601 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1602 "Only the names will be compared, without considering the set information." 1603 ) 1604 else: 1605 1606 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1607 1608 mask = [ 1609 full.upper() != x.upper() for x in self.agg_normalized_data.columns 1610 ] 1611 1612 if len([y for y in mask if y is False]) == 0: 1613 raise ValueError("Nothing found to reduce") 1614 1615 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 1616 1617 if self.agg_metadata is not None: 1618 1619 if inc_set: 1620 1621 self.agg_metadata["drop"] = ( 1622 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1623 ) 1624 1625 if "#" not in full: 1626 1627 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1628 1629 print( 1630 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1631 "Only the names will be compared, without considering the set information." 1632 ) 1633 else: 1634 1635 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1636 1637 mask = [full.upper() != x.upper() for x in self.agg_metadata["drop"]] 1638 1639 if len([y for y in mask if y is False]) == 0: 1640 raise ValueError("Nothing found to reduce") 1641 1642 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 1643 drop=True 1644 ) 1645 1646 self.agg_metadata = self.agg_metadata.drop( 1647 columns=["drop"], errors="ignore" 1648 ) 1649 1650 self.gene_calculation() 1651 self.cells_calculation() 1652 1653 def reduce_rows(self, features_list: list): 1654 """ 1655 Remove rows (features) whose names are included in features_list. 1656 1657 Parameters 1658 ---------- 1659 features_list : list 1660 List of features to search for in index/gene names; matching entries will be removed. 1661 1662 Update 1663 ------------ 1664 Mutates `self.input_data`, `self.normalized_data`, `self.input_metadata`, 1665 `self.agg_normalized_data`, and `self.agg_metadata` (if they exist), 1666 removing columns/rows that match `reg`. 1667 1668 Raises 1669 ------ 1670 Prints a message listing features that are not found in the data. 1671 """ 1672 1673 if self.input_data is not None: 1674 1675 res = find_features(self.input_data, features=features_list) 1676 1677 res_list = [x.upper() for x in res["included"]] 1678 1679 mask = [x.upper() not in res_list for x in self.input_data.index] 1680 1681 if len([y for y in mask if y is False]) == 0: 1682 raise ValueError("Nothing found to reduce") 1683 1684 self.input_data = self.input_data.loc[mask, :] 1685 1686 if self.normalized_data is not None: 1687 1688 res = find_features(self.normalized_data, features=features_list) 1689 1690 res_list = [x.upper() for x in res["included"]] 1691 1692 mask = [x.upper() not in res_list for x in self.normalized_data.index] 1693 1694 if len([y for y in mask if y is False]) == 0: 1695 raise ValueError("Nothing found to reduce") 1696 1697 self.normalized_data = self.normalized_data.loc[mask, :] 1698 1699 if self.agg_normalized_data is not None: 1700 1701 res = find_features(self.agg_normalized_data, features=features_list) 1702 1703 res_list = [x.upper() for x in res["included"]] 1704 1705 mask = [x.upper() not in res_list for x in self.agg_normalized_data.index] 1706 1707 if len([y for y in mask if y is False]) == 0: 1708 raise ValueError("Nothing found to reduce") 1709 1710 self.agg_normalized_data = self.agg_normalized_data.loc[mask, :] 1711 1712 if len(res["not_included"]) > 0: 1713 print("\nFeatures not found:") 1714 for i in res["not_included"]: 1715 print(i) 1716 1717 self.gene_calculation() 1718 self.cells_calculation() 1719 1720 def get_data(self, set_info: bool = False): 1721 """ 1722 Return normalized data with optional set annotation appended to column names. 1723 1724 Parameters 1725 ---------- 1726 set_info : bool, default False 1727 If True, column names are returned as "cell_name # set"; otherwise 1728 only the `cell_name` is used. 1729 1730 Returns 1731 ------- 1732 pandas.DataFrame 1733 The `self.normalized_data` table with columns renamed according to `set_info`. 1734 1735 Raises 1736 ------ 1737 AttributeError 1738 If `self.normalized_data` or `self.input_metadata` is missing. 1739 """ 1740 1741 to_return = self.normalized_data 1742 1743 if set_info: 1744 to_return.columns = ( 1745 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 1746 ) 1747 else: 1748 to_return.columns = self.input_metadata["cell_names"] 1749 1750 return to_return 1751 1752 def get_partial_data( 1753 self, 1754 names: list | str | None = None, 1755 features: list | str | None = None, 1756 name_slot: str = "cell_names", 1757 inc_metadata: bool = False, 1758 ): 1759 """ 1760 Return a subset of the data filtered by sample names and/or feature names. 1761 1762 Parameters 1763 ---------- 1764 names : list, str, or None 1765 Names of samples to include. If None, all samples are considered. 1766 1767 features : list, str, or None 1768 Names of features to include. If None, all features are considered. 1769 1770 name_slot : str 1771 Column in metadata to use as sample names. 1772 1773 inc_metadata : bool 1774 If True return tuple (data, metadata) 1775 1776 Returns 1777 ------- 1778 pandas.DataFrame 1779 Subset of the normalized data based on the specified names and features. 1780 """ 1781 1782 data = self.normalized_data.copy() 1783 metadata = self.input_metadata 1784 1785 if name_slot in self.input_metadata.columns: 1786 data.columns = self.input_metadata[name_slot] 1787 else: 1788 raise ValueError("'name_slot' not occured in data!'") 1789 1790 if isinstance(features, str): 1791 features = [features] 1792 elif features is None: 1793 features = [] 1794 1795 if isinstance(names, str): 1796 names = [names] 1797 elif names is None: 1798 names = [] 1799 1800 features = [x.upper() for x in features] 1801 names = [x.upper() for x in names] 1802 1803 columns_names = [x.upper() for x in data.columns] 1804 features_names = [x.upper() for x in data.index] 1805 1806 columns_bool = [True if x in names else False for x in columns_names] 1807 features_bool = [True if x in features else False for x in features_names] 1808 1809 if True not in columns_bool and True not in features_bool: 1810 print("Missing 'names' and/or 'features'. Returning full dataset instead.") 1811 1812 if True in columns_bool: 1813 data = data.loc[:, columns_bool] 1814 metadata = metadata.loc[columns_bool, :] 1815 1816 if True in features_bool: 1817 data = data.loc[features_bool, :] 1818 1819 not_in_features = [y for y in features if y not in features_names] 1820 1821 if len(not_in_features) > 0: 1822 print("\nThe following features were not found in data:") 1823 print("\n".join(not_in_features)) 1824 1825 not_in_names = [y for y in names if y not in columns_names] 1826 1827 if len(not_in_names) > 0: 1828 print("\nThe following names were not found in data:") 1829 print("\n".join(not_in_names)) 1830 1831 if inc_metadata: 1832 return data, metadata 1833 else: 1834 return data 1835 1836 def get_metadata(self): 1837 """ 1838 Return the stored input metadata. 1839 1840 Returns 1841 ------- 1842 pandas.DataFrame 1843 `self.input_metadata` (may be None if not set). 1844 """ 1845 1846 to_return = self.input_metadata 1847 1848 return to_return 1849 1850 def gene_calculation(self): 1851 """ 1852 Calculate and store per-cell counts (e.g., number of detected genes). 1853 1854 The method computes a binary (presence/absence) per cell and sums across 1855 features to produce `self.gene_calc`. 1856 1857 Update 1858 ------ 1859 Sets `self.gene_calc` as a pandas.Series. 1860 1861 Side Effects 1862 ------------ 1863 Uses `self.input_data` when available, otherwise `self.normalized_data`. 1864 """ 1865 1866 if self.input_data is not None: 1867 1868 bin_col = self.input_data.columns.copy() 1869 1870 bin_col = bin_col.where(bin_col <= 0, 1) 1871 1872 sum_data = bin_col.sum(axis=0) 1873 1874 self.gene_calc = sum_data 1875 1876 elif self.normalized_data is not None: 1877 1878 bin_col = self.normalized_data.copy() 1879 1880 bin_col = bin_col.where(bin_col <= 0, 1) 1881 1882 sum_data = bin_col.sum(axis=0) 1883 1884 self.gene_calc = sum_data 1885 1886 def gene_histograme(self, bins=100): 1887 """ 1888 Plot a histogram of the number of genes detected per cell. 1889 1890 Parameters 1891 ---------- 1892 bins : int, default 100 1893 Number of histogram bins. 1894 1895 Returns 1896 ------- 1897 matplotlib.figure.Figure 1898 Figure containing the histogram of gene contents. 1899 1900 Notes 1901 ----- 1902 Requires `self.gene_calc` to be computed prior to calling. 1903 """ 1904 1905 fig, ax = plt.subplots(figsize=(8, 5)) 1906 1907 _, bin_edges, _ = ax.hist( 1908 self.gene_calc, bins=bins, edgecolor="black", alpha=0.6 1909 ) 1910 1911 mu, sigma = np.mean(self.gene_calc), np.std(self.gene_calc) 1912 1913 x = np.linspace(min(self.gene_calc), max(self.gene_calc), 1000) 1914 y = norm.pdf(x, mu, sigma) 1915 1916 y_scaled = y * len(self.gene_calc) * (bin_edges[1] - bin_edges[0]) 1917 1918 ax.plot( 1919 x, y_scaled, "r-", linewidth=2, label=f"Normal(μ={mu:.2f}, σ={sigma:.2f})" 1920 ) 1921 1922 ax.set_xlabel("Value") 1923 ax.set_ylabel("Count") 1924 ax.set_title("Histogram of genes detected per cell") 1925 1926 ax.set_xticks(np.linspace(min(self.gene_calc), max(self.gene_calc), 20)) 1927 ax.tick_params(axis="x", rotation=90) 1928 1929 ax.legend() 1930 1931 return fig 1932 1933 def gene_threshold(self, min_n: int | None, max_n: int | None): 1934 """ 1935 Filter cells by gene-detection thresholds (min and/or max). 1936 1937 Parameters 1938 ---------- 1939 min_n : int or None 1940 Minimum number of detected genes required to keep a cell. 1941 1942 max_n : int or None 1943 Maximum number of detected genes allowed to keep a cell. 1944 1945 Update 1946 ------- 1947 Filters `self.input_data`, `self.normalized_data`, `self.input_metadata` 1948 (and calls `average()` if `self.agg_normalized_data` exists). 1949 1950 Side Effects 1951 ------------ 1952 Raises ValueError if both bounds are None or if filtering removes all cells. 1953 """ 1954 1955 if min_n is not None and max_n is not None: 1956 mask = (self.gene_calc > min_n) & (self.gene_calc < max_n) 1957 elif min_n is None and max_n is not None: 1958 mask = self.gene_calc < max_n 1959 elif min_n is not None and max_n is None: 1960 mask = self.gene_calc > min_n 1961 else: 1962 raise ValueError("Lack of both min_n and max_n values") 1963 1964 if self.input_data is not None: 1965 1966 if len([y for y in mask if y is False]) == 0: 1967 raise ValueError("Nothing to reduce") 1968 1969 self.input_data = self.input_data.loc[:, mask.values] 1970 1971 if self.normalized_data is not None: 1972 1973 if len([y for y in mask if y is False]) == 0: 1974 raise ValueError("Nothing to reduce") 1975 1976 self.normalized_data = self.normalized_data.loc[:, mask.values] 1977 1978 if self.input_metadata is not None: 1979 1980 if len([y for y in mask if y is False]) == 0: 1981 raise ValueError("Nothing to reduce") 1982 1983 self.input_metadata = self.input_metadata.loc[mask.values, :].reset_index( 1984 drop=True 1985 ) 1986 1987 self.input_metadata = self.input_metadata.drop( 1988 columns=["drop"], errors="ignore" 1989 ) 1990 1991 if self.agg_normalized_data is not None: 1992 self.average() 1993 1994 self.gene_calculation() 1995 self.cells_calculation() 1996 1997 def cells_calculation(self, name_slot="cell_names"): 1998 """ 1999 Calculate number of cells per call name / cluster. 2000 2001 The method computes a binary (presence/absence) per cell name / cluster and sums across 2002 cells. 2003 2004 Parameters 2005 ---------- 2006 name_slot : str, default 'cell_names' 2007 Column in metadata to use as sample names. 2008 2009 Update 2010 ------ 2011 Sets `self.cells_calc` as a pd.DataFrame. 2012 """ 2013 2014 ls = list(self.input_metadata[name_slot]) 2015 2016 df = pd.DataFrame( 2017 { 2018 "cluster": pd.Series(ls).value_counts().index, 2019 "n": pd.Series(ls).value_counts().values, 2020 } 2021 ) 2022 2023 self.cells_calc = df 2024 2025 def cell_histograme(self, name_slot: str = "cell_names"): 2026 """ 2027 Plot a histogram of the number of cells detected per cell name (cluster). 2028 2029 Parameters 2030 ---------- 2031 name_slot : str, default 'cell_names' 2032 Column in metadata to use as sample names. 2033 2034 Returns 2035 ------- 2036 matplotlib.figure.Figure 2037 Figure containing the histogram of cell contents. 2038 2039 Notes 2040 ----- 2041 Requires `self.cells_calc` to be computed prior to calling. 2042 """ 2043 2044 if name_slot != "cell_names": 2045 self.cells_calculation(name_slot=name_slot) 2046 2047 fig, ax = plt.subplots(figsize=(8, 5)) 2048 2049 _, bin_edges, _ = ax.hist( 2050 list(self.cells_calc["n"]), 2051 bins=len(set(self.cells_calc["cluster"])), 2052 edgecolor="black", 2053 color="orange", 2054 alpha=0.6, 2055 ) 2056 2057 mu, sigma = np.mean(list(self.cells_calc["n"])), np.std( 2058 list(self.cells_calc["n"]) 2059 ) 2060 2061 x = np.linspace( 2062 min(list(self.cells_calc["n"])), max(list(self.cells_calc["n"])), 1000 2063 ) 2064 y = norm.pdf(x, mu, sigma) 2065 2066 y_scaled = y * len(list(self.cells_calc["n"])) * (bin_edges[1] - bin_edges[0]) 2067 2068 ax.plot( 2069 x, y_scaled, "r-", linewidth=2, label=f"Normal(μ={mu:.2f}, σ={sigma:.2f})" 2070 ) 2071 2072 ax.set_xlabel("Value") 2073 ax.set_ylabel("Count") 2074 ax.set_title("Histogram of cells detected per cell name / cluster") 2075 2076 ax.set_xticks( 2077 np.linspace( 2078 min(list(self.cells_calc["n"])), max(list(self.cells_calc["n"])), 20 2079 ) 2080 ) 2081 ax.tick_params(axis="x", rotation=90) 2082 2083 ax.legend() 2084 2085 return fig 2086 2087 def cluster_threshold(self, min_n: int | None, name_slot: str = "cell_names"): 2088 """ 2089 Filter cell names / clusters by cell-detection threshold. 2090 2091 Parameters 2092 ---------- 2093 min_n : int or None 2094 Minimum number of detected genes required to keep a cell. 2095 2096 name_slot : str, default 'cell_names' 2097 Column in metadata to use as sample names. 2098 2099 2100 Update 2101 ------- 2102 Filters `self.input_data`, `self.normalized_data`, `self.input_metadata` 2103 (and calls `average()` if `self.agg_normalized_data` exists). 2104 """ 2105 2106 if name_slot != "cell_names": 2107 self.cells_calculation(name_slot=name_slot) 2108 2109 if min_n is not None: 2110 names = self.cells_calc["cluster"][self.cells_calc["n"] < min_n] 2111 else: 2112 raise ValueError("Lack of min_n value") 2113 2114 if len(names) > 0: 2115 2116 if self.input_data is not None: 2117 2118 self.input_data.columns = self.input_metadata[name_slot] 2119 2120 mask = [not any(r in x for r in names) for x in self.input_data.columns] 2121 2122 if len([y for y in mask if y is False]) > 0: 2123 2124 self.input_data = self.input_data.loc[:, mask] 2125 2126 if self.normalized_data is not None: 2127 2128 self.normalized_data.columns = self.input_metadata[name_slot] 2129 2130 mask = [ 2131 not any(r in x for r in names) for x in self.normalized_data.columns 2132 ] 2133 2134 if len([y for y in mask if y is False]) > 0: 2135 2136 self.normalized_data = self.normalized_data.loc[:, mask] 2137 2138 if self.input_metadata is not None: 2139 2140 self.input_metadata["drop"] = self.input_metadata[name_slot] 2141 2142 mask = [ 2143 not any(r in x for r in names) for x in self.input_metadata["drop"] 2144 ] 2145 2146 if len([y for y in mask if y is False]) > 0: 2147 2148 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 2149 drop=True 2150 ) 2151 2152 self.input_metadata = self.input_metadata.drop( 2153 columns=["drop"], errors="ignore" 2154 ) 2155 2156 if self.agg_normalized_data is not None: 2157 2158 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 2159 2160 mask = [ 2161 not any(r in x for r in names) 2162 for x in self.agg_normalized_data.columns 2163 ] 2164 2165 if len([y for y in mask if y is False]) > 0: 2166 2167 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 2168 2169 if self.agg_metadata is not None: 2170 2171 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 2172 2173 mask = [ 2174 not any(r in x for r in names) for x in self.agg_metadata["drop"] 2175 ] 2176 2177 if len([y for y in mask if y is False]) > 0: 2178 2179 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 2180 drop=True 2181 ) 2182 2183 self.agg_metadata = self.agg_metadata.drop( 2184 columns=["drop"], errors="ignore" 2185 ) 2186 2187 self.gene_calculation() 2188 self.cells_calculation() 2189 2190 def load_sparse_from_projects(self, normalized_data: bool = False): 2191 """ 2192 Load sparse 10x-style datasets from stored project paths, concatenate them, 2193 and populate `input_data` / `normalized_data` and `input_metadata`. 2194 2195 Parameters 2196 ---------- 2197 normalized_data : bool, default False 2198 If True, store concatenated tables in `self.normalized_data`. 2199 If False, store them in `self.input_data` and normalization 2200 is needed using normalize_data() method. 2201 2202 Side Effects 2203 ------------ 2204 - Reads each project using `load_sparse(...)` (expects matrix.mtx, genes.tsv, barcodes.tsv). 2205 - Concatenates all projects column-wise and sets `self.input_metadata`. 2206 - Replaces NaNs with zeros and updates `self.gene_calc`. 2207 """ 2208 2209 obj = self.objects 2210 2211 full_data = pd.DataFrame() 2212 full_metadata = pd.DataFrame() 2213 2214 for ke in obj.keys(): 2215 print(ke) 2216 2217 dt, met = load_sparse(path=obj[ke], name=ke) 2218 2219 full_data = pd.concat([full_data, dt], axis=1) 2220 full_metadata = pd.concat([full_metadata, met], axis=0) 2221 2222 full_data[np.isnan(full_data)] = 0 2223 2224 if normalized_data: 2225 self.normalized_data = full_data 2226 self.input_metadata = full_metadata 2227 else: 2228 2229 self.input_data = full_data 2230 self.input_metadata = full_metadata 2231 2232 self.gene_calculation() 2233 self.cells_calculation() 2234 2235 def rename_names(self, mapping: dict, slot: str = "cell_names"): 2236 """ 2237 Rename entries in `self.input_metadata[slot]` according to a provided mapping. 2238 2239 Parameters 2240 ---------- 2241 mapping : dict 2242 Dictionary with keys 'old_name' and 'new_name', each mapping to a list 2243 of equal length describing replacements. 2244 2245 slot : str, default 'cell_names' 2246 Metadata column to operate on. 2247 2248 Update 2249 ------- 2250 Updates `self.input_metadata[slot]` in-place with renamed values. 2251 2252 Raises 2253 ------ 2254 ValueError 2255 If mapping keys are incorrect, lengths differ, or some 'old_name' values 2256 are not present in the metadata column. 2257 """ 2258 2259 if set(["old_name", "new_name"]) != set(mapping.keys()): 2260 raise ValueError( 2261 "Mapping dictionary must contain keys 'old_name' and 'new_name', " 2262 "each with a list of names to change." 2263 ) 2264 2265 if len(mapping["old_name"]) != len(mapping["new_name"]): 2266 raise ValueError( 2267 "Mapping dictionary lists 'old_name' and 'new_name' " 2268 "must have the same length!" 2269 ) 2270 2271 names_vector = list(self.input_metadata[slot]) 2272 2273 if not all(elem in names_vector for elem in list(mapping["old_name"])): 2274 raise ValueError( 2275 f"Some entries from 'old_name' do not exist in the names of slot {slot}." 2276 ) 2277 2278 replace_dict = dict(zip(mapping["old_name"], mapping["new_name"])) 2279 2280 names_vector_ret = [replace_dict.get(item, item) for item in names_vector] 2281 2282 self.input_metadata[slot] = names_vector_ret 2283 2284 def rename_subclusters(self, mapping): 2285 """ 2286 Rename labels stored in `self.subclusters_.subclusters` according to mapping. 2287 2288 Parameters 2289 ---------- 2290 mapping : dict 2291 Mapping with keys 'old_name' and 'new_name' (lists of equal length). 2292 2293 Update 2294 ------- 2295 Updates `self.subclusters_.subclusters` with renamed labels. 2296 2297 Raises 2298 ------ 2299 ValueError 2300 If mapping is invalid or old names are not present. 2301 """ 2302 2303 if set(["old_name", "new_name"]) != set(mapping.keys()): 2304 raise ValueError( 2305 "Mapping dictionary must contain keys 'old_name' and 'new_name', " 2306 "each with a list of names to change." 2307 ) 2308 2309 if len(mapping["old_name"]) != len(mapping["new_name"]): 2310 raise ValueError( 2311 "Mapping dictionary lists 'old_name' and 'new_name' " 2312 "must have the same length!" 2313 ) 2314 2315 names_vector = list(self.subclusters_.subclusters) 2316 2317 if not all(elem in names_vector for elem in list(mapping["old_name"])): 2318 raise ValueError( 2319 "Some entries from 'old_name' do not exist in the subcluster names." 2320 ) 2321 2322 replace_dict = dict(zip(mapping["old_name"], mapping["new_name"])) 2323 2324 names_vector_ret = [replace_dict.get(item, item) for item in names_vector] 2325 2326 self.subclusters_.subclusters = names_vector_ret 2327 2328 def save_sparse( 2329 self, 2330 path_to_save: str = os.getcwd(), 2331 name_slot: str = "cell_names", 2332 data_slot: str = "normalized", 2333 ): 2334 """ 2335 Export data as 10x-compatible sparse files (matrix.mtx, barcodes.tsv, genes.tsv). 2336 2337 Parameters 2338 ---------- 2339 path_to_save : str, default current working directory 2340 Directory where files will be written. 2341 2342 name_slot : str, default 'cell_names' 2343 Metadata column providing cell names for barcodes.tsv. 2344 2345 data_slot : str, default 'normalized' 2346 Either 'normalized' (uses self.normalized_data) or 'count' (uses self.input_data). 2347 2348 Raises 2349 ------ 2350 ValueError 2351 If `data_slot` is not 'normalized' or 'count'. 2352 """ 2353 2354 names = self.input_metadata[name_slot] 2355 2356 if data_slot.lower() == "normalized": 2357 2358 features = list(self.normalized_data.index) 2359 mtx = sparse.csr_matrix(self.normalized_data) 2360 2361 elif data_slot.lower() == "count": 2362 2363 features = list(self.input_data.index) 2364 mtx = sparse.csr_matrix(self.input_data) 2365 2366 else: 2367 raise ValueError("'data_slot' must be included in 'normalized' or 'count'") 2368 2369 os.makedirs(path_to_save, exist_ok=True) 2370 2371 mmwrite(os.path.join(path_to_save, "matrix.mtx"), mtx) 2372 2373 pd.Series(names).to_csv( 2374 os.path.join(path_to_save, "barcodes.tsv"), 2375 index=False, 2376 header=False, 2377 sep="\t", 2378 ) 2379 2380 pd.Series(features).to_csv( 2381 os.path.join(path_to_save, "genes.tsv"), index=False, header=False, sep="\t" 2382 ) 2383 2384 def normalize_counts( 2385 self, normalize_factor: int = 100000, log_transform: bool = True 2386 ): 2387 """ 2388 Normalize raw counts to counts-per-(normalize_factor) 2389 (e.g., CPM, TPM - depending on normalize_factor). 2390 2391 Parameters 2392 ---------- 2393 normalize_factor : int, default 100000 2394 Scaling factor used after dividing by column sums. 2395 2396 log_transform : bool, default True 2397 If True, apply log2(x+1) transformation to normalized values. 2398 2399 Update 2400 ------- 2401 Sets `self.normalized_data` to normalized values (fills NaNs with 0). 2402 2403 Raises 2404 ------ 2405 ValueError 2406 If `self.input_data` is missing (cannot normalize). 2407 """ 2408 if self.input_data is None: 2409 raise ValueError("Input data is missing, cannot normalize.") 2410 2411 sum_col = self.input_data.sum() 2412 self.normalized_data = self.input_data.div(sum_col).fillna(0) * normalize_factor 2413 2414 if log_transform: 2415 # log2(x + 1) to avoid -inf for zeros 2416 self.normalized_data = np.log2(self.normalized_data + 1) 2417 2418 def statistic( 2419 self, 2420 cells=None, 2421 sets=None, 2422 min_exp: float = 0.01, 2423 min_pct: float = 0.1, 2424 n_proc: int = 10, 2425 ): 2426 """ 2427 Compute per-feature statistics (Mann–Whitney U) comparing target vs rest. 2428 2429 This is a wrapper similar to `calc_DEG` tailored to use `self.normalized_data` 2430 and `self.input_metadata`. It returns per-feature statistics including p-values, 2431 adjusted p-values, means, variances, effect-size measures and fold-changes. 2432 2433 Parameters 2434 ---------- 2435 cells : list, 'All', dict, or None 2436 Defines the target cells or groups for comparison (several modes supported). 2437 2438 sets : 'All', dict, or None 2439 Alternative grouping mode (operate on `self.input_metadata['sets']`). 2440 2441 min_exp : float, default 0.01 2442 Minimum expression threshold used when filtering features. 2443 2444 min_pct : float, default 0.1 2445 Minimum proportion of expressing cells in the target group required to test a feature. 2446 2447 n_proc : int, default 10 2448 Number of parallel jobs to use. 2449 2450 Returns 2451 ------- 2452 pandas.DataFrame or dict 2453 Results DataFrame (or dict containing valid/control cells + DataFrame), 2454 similar to `calc_DEG` interface. 2455 2456 Raises 2457 ------ 2458 ValueError 2459 If neither `cells` nor `sets` is provided, or input metadata mismatch occurs. 2460 2461 Notes 2462 ----- 2463 Multiple modes supported: single-list entities, 'All', pairwise dicts, etc. 2464 """ 2465 2466 offset = 1e-100 2467 2468 def stat_calc(choose, feature_name): 2469 target_values = choose.loc[choose["DEG"] == "target", feature_name] 2470 rest_values = choose.loc[choose["DEG"] == "rest", feature_name] 2471 2472 pct_valid = (target_values > 0).sum() / len(target_values) 2473 pct_rest = (rest_values > 0).sum() / len(rest_values) 2474 2475 avg_valid = np.mean(target_values) 2476 avg_ctrl = np.mean(rest_values) 2477 sd_valid = np.std(target_values, ddof=1) 2478 sd_ctrl = np.std(rest_values, ddof=1) 2479 esm = (avg_valid - avg_ctrl) / np.sqrt(((sd_valid**2 + sd_ctrl**2) / 2)) 2480 2481 if np.sum(target_values) == np.sum(rest_values): 2482 p_val = 1.0 2483 else: 2484 _, p_val = stats.mannwhitneyu( 2485 target_values, rest_values, alternative="two-sided" 2486 ) 2487 2488 return { 2489 "feature": feature_name, 2490 "p_val": p_val, 2491 "pct_valid": pct_valid, 2492 "pct_ctrl": pct_rest, 2493 "avg_valid": avg_valid, 2494 "avg_ctrl": avg_ctrl, 2495 "sd_valid": sd_valid, 2496 "sd_ctrl": sd_ctrl, 2497 "esm": esm, 2498 } 2499 2500 def prepare_and_run_stat(choose, valid_group, min_exp, min_pct, n_proc): 2501 2502 def safe_min_half(series): 2503 filtered = series[(series > ((2**-1074) * 2)) & (series.notna())] 2504 return filtered.min() / 2 if not filtered.empty else 0 2505 2506 tmp_dat = choose[choose["DEG"] == "target"] 2507 tmp_dat = tmp_dat.drop("DEG", axis=1) 2508 2509 counts = (tmp_dat > min_exp).sum(axis=0) 2510 2511 total_count = tmp_dat.shape[0] 2512 2513 info = pd.DataFrame( 2514 {"feature": list(tmp_dat.columns), "pct": list(counts / total_count)} 2515 ) 2516 2517 del tmp_dat 2518 2519 drop_col = info["feature"][info["pct"] <= min_pct] 2520 2521 if len(drop_col) + 1 == len(choose.columns): 2522 drop_col = info["feature"][info["pct"] == 0] 2523 2524 del info 2525 2526 choose = choose.drop(list(drop_col), axis=1) 2527 2528 results = Parallel(n_jobs=n_proc)( 2529 delayed(stat_calc)(choose, feature) 2530 for feature in tqdm(choose.columns[choose.columns != "DEG"]) 2531 ) 2532 2533 df = pd.DataFrame(results) 2534 df = df[(df["avg_valid"] > 0) | (df["avg_ctrl"] > 0)] 2535 2536 df["valid_group"] = valid_group 2537 df.sort_values(by="p_val", inplace=True) 2538 2539 num_tests = len(df) 2540 df["adj_pval"] = np.minimum( 2541 1, (df["p_val"] * num_tests) / np.arange(1, num_tests + 1) 2542 ) 2543 2544 valid_factor = safe_min_half(df["avg_valid"]) 2545 ctrl_factor = safe_min_half(df["avg_ctrl"]) 2546 2547 cv_factor = min(valid_factor, ctrl_factor) 2548 2549 if cv_factor == 0: 2550 cv_factor = max(valid_factor, ctrl_factor) 2551 2552 if not np.isfinite(cv_factor) or cv_factor == 0: 2553 cv_factor += offset 2554 2555 valid = df["avg_valid"].where( 2556 df["avg_valid"] != 0, df["avg_valid"] + cv_factor 2557 ) 2558 ctrl = df["avg_ctrl"].where(df["avg_ctrl"] != 0, df["avg_ctrl"] + cv_factor) 2559 2560 df["FC"] = valid / ctrl 2561 2562 df["log(FC)"] = np.log2(df["FC"]) 2563 df["norm_diff"] = df["avg_valid"] - df["avg_ctrl"] 2564 2565 return df 2566 2567 choose = self.normalized_data.copy().T 2568 2569 final_results = [] 2570 2571 if isinstance(cells, list) and sets is None: 2572 print("\nAnalysis started...\nComparing selected cells to the whole set...") 2573 choose.index = ( 2574 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2575 ) 2576 2577 if "#" not in cells[0]: 2578 choose.index = self.input_metadata["cell_names"] 2579 2580 print( 2581 "Not include the set info (name # set) in the 'cells' list. " 2582 "Only the names will be compared, without considering the set information." 2583 ) 2584 2585 labels = ["target" if idx in cells else "rest" for idx in choose.index] 2586 valid = list( 2587 set(choose.index[[i for i, x in enumerate(labels) if x == "target"]]) 2588 ) 2589 2590 choose["DEG"] = labels 2591 choose = choose[choose["DEG"] != "drop"] 2592 2593 result_df = prepare_and_run_stat( 2594 choose.reset_index(drop=True), 2595 valid_group=valid, 2596 min_exp=min_exp, 2597 min_pct=min_pct, 2598 n_proc=n_proc, 2599 ) 2600 return {"valid_cells": valid, "control_cells": "rest", "DEG": result_df} 2601 2602 elif cells == "All" and sets is None: 2603 print("\nAnalysis started...\nComparing each type of cell to others...") 2604 choose.index = ( 2605 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2606 ) 2607 unique_labels = set(choose.index) 2608 2609 for label in tqdm(unique_labels): 2610 print(f"\nCalculating statistics for {label}") 2611 labels = ["target" if idx == label else "rest" for idx in choose.index] 2612 choose["DEG"] = labels 2613 choose = choose[choose["DEG"] != "drop"] 2614 result_df = prepare_and_run_stat( 2615 choose.copy(), 2616 valid_group=label, 2617 min_exp=min_exp, 2618 min_pct=min_pct, 2619 n_proc=n_proc, 2620 ) 2621 final_results.append(result_df) 2622 2623 return pd.concat(final_results, ignore_index=True) 2624 2625 elif cells is None and sets == "All": 2626 print("\nAnalysis started...\nComparing each set/group to others...") 2627 choose.index = self.input_metadata["sets"] 2628 unique_sets = set(choose.index) 2629 2630 for label in tqdm(unique_sets): 2631 print(f"\nCalculating statistics for {label}") 2632 labels = ["target" if idx == label else "rest" for idx in choose.index] 2633 2634 choose["DEG"] = labels 2635 choose = choose[choose["DEG"] != "drop"] 2636 result_df = prepare_and_run_stat( 2637 choose.copy(), 2638 valid_group=label, 2639 min_exp=min_exp, 2640 min_pct=min_pct, 2641 n_proc=n_proc, 2642 ) 2643 final_results.append(result_df) 2644 2645 return pd.concat(final_results, ignore_index=True) 2646 2647 elif cells is None and isinstance(sets, dict): 2648 print("\nAnalysis started...\nComparing groups...") 2649 2650 choose.index = self.input_metadata["sets"] 2651 2652 group_list = list(sets.keys()) 2653 if len(group_list) != 2: 2654 print("Only pairwise group comparison is supported.") 2655 return None 2656 2657 labels = [ 2658 ( 2659 "target" 2660 if idx in sets[group_list[0]] 2661 else "rest" if idx in sets[group_list[1]] else "drop" 2662 ) 2663 for idx in choose.index 2664 ] 2665 choose["DEG"] = labels 2666 choose = choose[choose["DEG"] != "drop"] 2667 2668 result_df = prepare_and_run_stat( 2669 choose.reset_index(drop=True), 2670 valid_group=group_list[0], 2671 min_exp=min_exp, 2672 min_pct=min_pct, 2673 n_proc=n_proc, 2674 ) 2675 return result_df 2676 2677 elif isinstance(cells, dict) and sets is None: 2678 print("\nAnalysis started...\nComparing groups...") 2679 choose.index = ( 2680 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2681 ) 2682 2683 if "#" not in cells[list(cells.keys())[0]][0]: 2684 choose.index = self.input_metadata["cell_names"] 2685 2686 print( 2687 "Not include the set info (name # set) in the 'cells' dict. " 2688 "Only the names will be compared, without considering the set information." 2689 ) 2690 2691 group_list = list(cells.keys()) 2692 if len(group_list) != 2: 2693 print("Only pairwise group comparison is supported.") 2694 return None 2695 2696 labels = [ 2697 ( 2698 "target" 2699 if idx in cells[group_list[0]] 2700 else "rest" if idx in cells[group_list[1]] else "drop" 2701 ) 2702 for idx in choose.index 2703 ] 2704 2705 choose["DEG"] = labels 2706 choose = choose[choose["DEG"] != "drop"] 2707 2708 result_df = prepare_and_run_stat( 2709 choose.reset_index(drop=True), 2710 valid_group=group_list[0], 2711 min_exp=min_exp, 2712 min_pct=min_pct, 2713 n_proc=n_proc, 2714 ) 2715 2716 return result_df.reset_index(drop=True) 2717 2718 else: 2719 raise ValueError( 2720 "You must specify either 'cells' or 'sets' (or both). None were provided, which is not allowed for this analysis." 2721 ) 2722 2723 def calculate_difference_markers( 2724 self, min_exp=0, min_pct=0.25, n_proc=10, force: bool = False 2725 ): 2726 """ 2727 Compute differential markers (var_data) if not already present. 2728 2729 Parameters 2730 ---------- 2731 min_exp : float, default 0 2732 Minimum expression threshold passed to `statistic`. 2733 2734 min_pct : float, default 0.25 2735 Minimum percent expressed in target group. 2736 2737 n_proc : int, default 10 2738 Parallel jobs. 2739 2740 force : bool, default False 2741 If True, recompute even if `self.var_data` is present. 2742 2743 Update 2744 ------- 2745 Sets `self.var_data` to the result of `self.statistic(...)`. 2746 2747 Raise 2748 ------ 2749 ValueError if already computed and `force` is False. 2750 """ 2751 2752 if self.var_data is None or force: 2753 2754 self.var_data = self.statistic( 2755 cells="All", sets=None, min_exp=min_exp, min_pct=min_pct, n_proc=n_proc 2756 ) 2757 2758 else: 2759 raise ValueError( 2760 "self.calculate_difference_markers() has already been executed. " 2761 "The results are stored in self.var. " 2762 "If you want to recalculate with different statistics, please rerun the method with force=True." 2763 ) 2764 2765 def clustering_features( 2766 self, 2767 features_list: list | None, 2768 name_slot: str = "cell_names", 2769 p_val: float = 0.05, 2770 top_n: int = 25, 2771 adj_mean: bool = True, 2772 beta: float = 0.2, 2773 ): 2774 """ 2775 Prepare clustering input by selecting marker features and optionally smoothing cell values 2776 toward group means. 2777 2778 Parameters 2779 ---------- 2780 features_list : list or None 2781 If provided, use this list of features. If None, features are selected 2782 from `self.var_data` (adj_pval <= p_val, positive logFC) picking `top_n` per group. 2783 2784 name_slot : str, default 'cell_names' 2785 Metadata column used for naming. 2786 2787 p_val : float, default 0.05 2788 Adjusted p-value cutoff when selecting features automatically. 2789 2790 top_n : int, default 25 2791 Number of top features per valid group to keep if `features_list` is None. 2792 2793 adj_mean : bool, default True 2794 If True, adjust cell values toward group means using `beta`. 2795 2796 beta : float, default 0.2 2797 Adjustment strength toward group mean. 2798 2799 Update 2800 ------ 2801 Sets `self.clustering_data` and `self.clustering_metadata` to the selected subset, 2802 ready for PCA/UMAP/clustering. 2803 """ 2804 2805 if features_list is None or len(features_list) == 0: 2806 2807 if self.var_data is None: 2808 raise ValueError( 2809 "Lack of 'self.var_data'. Use self.calculate_difference_markers() method first." 2810 ) 2811 2812 df_tmp = self.var_data[self.var_data["adj_pval"] <= p_val] 2813 df_tmp = df_tmp[df_tmp["log(FC)"] > 0] 2814 df_tmp = ( 2815 df_tmp.sort_values( 2816 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 2817 ) 2818 .groupby("valid_group") 2819 .head(top_n) 2820 ) 2821 2822 feaures_list = list(set(df_tmp["feature"])) 2823 2824 data = self.get_partial_data( 2825 names=None, features=feaures_list, name_slot=name_slot 2826 ) 2827 data_avg = average(data) 2828 2829 if adj_mean: 2830 data = adjust_cells_to_group_mean(data=data, data_avg=data_avg, beta=beta) 2831 2832 self.clustering_data = data 2833 2834 self.clustering_metadata = self.input_metadata 2835 2836 def average(self): 2837 """ 2838 Aggregate normalized data by (cell_name, set) pairs computing the mean per group. 2839 2840 The method constructs new column names as "cell_name # set", averages columns 2841 sharing identical labels, and populates `self.agg_normalized_data` and `self.agg_metadata`. 2842 2843 Update 2844 ------ 2845 Sets `self.agg_normalized_data` (features x aggregated samples) and 2846 `self.agg_metadata` (DataFrame with 'cell_names' and 'sets'). 2847 """ 2848 2849 wide_data = self.normalized_data 2850 2851 wide_metadata = self.input_metadata 2852 2853 new_names = wide_metadata["cell_names"] + " # " + wide_metadata["sets"] 2854 2855 wide_data.columns = list(new_names) 2856 2857 aggregated_df = wide_data.T.groupby(level=0).mean().T 2858 2859 sets = [re.sub(".*# ", "", x) for x in aggregated_df.columns] 2860 names = [re.sub(" #.*", "", x) for x in aggregated_df.columns] 2861 2862 aggregated_df.columns = names 2863 aggregated_metadata = pd.DataFrame({"cell_names": names, "sets": sets}) 2864 2865 self.agg_metadata = aggregated_metadata 2866 self.agg_normalized_data = aggregated_df 2867 2868 def estimating_similarity( 2869 self, method="pearson", p_val: float = 0.05, top_n: int = 25 2870 ): 2871 """ 2872 Estimate pairwise similarity and Euclidean distance between aggregated samples. 2873 2874 Parameters 2875 ---------- 2876 method : str, default 'pearson' 2877 Correlation method to use (passed to pandas.DataFrame.corr()). 2878 2879 p_val : float, default 0.05 2880 Adjusted p-value cutoff used to select marker features from `self.var_data`. 2881 2882 top_n : int, default 25 2883 Number of top features per valid group to include. 2884 2885 Update 2886 ------- 2887 Computes a combined table with per-pair correlation and euclidean distance 2888 and stores it in `self.similarity`. 2889 """ 2890 2891 if self.var_data is None: 2892 raise ValueError( 2893 "Lack of 'self.var_data'. Use self.calculate_difference_markers() method first." 2894 ) 2895 2896 if self.agg_normalized_data is None: 2897 self.average() 2898 2899 metadata = self.agg_metadata 2900 data = self.agg_normalized_data 2901 2902 df_tmp = self.var_data[self.var_data["adj_pval"] <= p_val] 2903 df_tmp = df_tmp[df_tmp["log(FC)"] > 0] 2904 df_tmp = ( 2905 df_tmp.sort_values( 2906 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 2907 ) 2908 .groupby("valid_group") 2909 .head(top_n) 2910 ) 2911 2912 data = data.loc[list(set(df_tmp["feature"]))] 2913 2914 if len(set(metadata["sets"])) > 1: 2915 data.columns = data.columns + " # " + [x for x in metadata["sets"]] 2916 else: 2917 data = data.copy() 2918 2919 scaler = StandardScaler() 2920 2921 scaled_data = scaler.fit_transform(data) 2922 2923 scaled_df = pd.DataFrame(scaled_data, columns=data.columns) 2924 2925 cor = scaled_df.corr(method=method) 2926 cor_df = cor.stack().reset_index() 2927 cor_df.columns = ["cell1", "cell2", "correlation"] 2928 2929 distances = pdist(scaled_df.T, metric="euclidean") 2930 dist_mat = pd.DataFrame( 2931 squareform(distances), index=scaled_df.columns, columns=scaled_df.columns 2932 ) 2933 dist_df = dist_mat.stack().reset_index() 2934 dist_df.columns = ["cell1", "cell2", "euclidean_dist"] 2935 2936 full = pd.merge(cor_df, dist_df, on=["cell1", "cell2"]) 2937 2938 full = full[full["cell1"] != full["cell2"]] 2939 full = full.reset_index(drop=True) 2940 2941 self.similarity = full 2942 2943 def similarity_plot( 2944 self, 2945 split_sets=True, 2946 set_info: bool = True, 2947 cmap="seismic", 2948 width=12, 2949 height=10, 2950 ): 2951 """ 2952 Visualize similarity as a dot plot. 2953 2954 Parameters 2955 ---------- 2956 split_sets : bool, default True 2957 If True and set information is present, split plotting area roughly into two halves to visualize cross-set pairs. 2958 2959 set_info : bool, default True 2960 If True, keep the ' # set' annotation in labels; otherwise strip it. 2961 2962 cmap : str, default 'seismic' 2963 Color map for correlation (hue). 2964 2965 width : int, default 12 2966 Figure width. 2967 2968 height : int, default 10 2969 Figure height. 2970 2971 Returns 2972 ------- 2973 matplotlib.figure.Figure 2974 2975 Raises 2976 ------ 2977 ValueError 2978 If `self.similarity` is None. 2979 2980 Notes 2981 ----- 2982 The function filters pairs by z-scored euclidean distance > 0 to focus on closer pairs. 2983 """ 2984 2985 if self.similarity is None: 2986 raise ValueError( 2987 "Similarity data is missing. Please calculate similarity using self.estimating_similarity." 2988 ) 2989 2990 similarity_data = self.similarity 2991 2992 if " # " in similarity_data["cell1"][0]: 2993 similarity_data["set1"] = [ 2994 re.sub(".*# ", "", x) for x in similarity_data["cell1"] 2995 ] 2996 similarity_data["set2"] = [ 2997 re.sub(".*# ", "", x) for x in similarity_data["cell2"] 2998 ] 2999 3000 if split_sets and " # " in similarity_data["cell1"][0]: 3001 sets = list( 3002 set(list(similarity_data["set1"]) + list(similarity_data["set2"])) 3003 ) 3004 3005 mm = math.ceil(len(sets) / 2) 3006 3007 x_s = sets[0:mm] 3008 y_s = sets[mm : len(sets)] 3009 3010 similarity_data = similarity_data[similarity_data["set1"].isin(x_s)] 3011 similarity_data = similarity_data[similarity_data["set2"].isin(y_s)] 3012 3013 similarity_data = similarity_data.sort_values(["set1", "set2"]) 3014 3015 if set_info is False and " # " in similarity_data["cell1"][0]: 3016 similarity_data["cell1"] = [ 3017 re.sub(" #.*", "", x) for x in similarity_data["cell1"] 3018 ] 3019 similarity_data["cell2"] = [ 3020 re.sub(" #.*", "", x) for x in similarity_data["cell2"] 3021 ] 3022 3023 similarity_data["-euclidean_zscore"] = -zscore( 3024 similarity_data["euclidean_dist"] 3025 ) 3026 3027 similarity_data = similarity_data[similarity_data["-euclidean_zscore"] > 0] 3028 3029 fig = plt.figure(figsize=(width, height)) 3030 sns.scatterplot( 3031 data=similarity_data, 3032 x="cell1", 3033 y="cell2", 3034 hue="correlation", 3035 size="-euclidean_zscore", 3036 sizes=(1, 100), 3037 palette=cmap, 3038 alpha=1, 3039 edgecolor="black", 3040 ) 3041 3042 plt.xticks(rotation=90) 3043 plt.yticks(rotation=0) 3044 plt.xlabel("Cell 1") 3045 plt.ylabel("Cell 2") 3046 plt.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 3047 3048 plt.grid(True, alpha=0.6) 3049 3050 plt.tight_layout() 3051 3052 return fig 3053 3054 def spatial_similarity( 3055 self, 3056 set_info: bool = True, 3057 bandwidth=1, 3058 n_neighbors=5, 3059 min_dist=0.1, 3060 legend_split=2, 3061 point_size=100, 3062 spread=1.0, 3063 set_op_mix_ratio=1.0, 3064 local_connectivity=1, 3065 repulsion_strength=1.0, 3066 negative_sample_rate=5, 3067 threshold=0.1, 3068 width=12, 3069 height=10, 3070 ): 3071 """ 3072 Create a spatial UMAP-like visualization of similarity relationships between samples. 3073 3074 Parameters 3075 ---------- 3076 set_info : bool, default True 3077 If True, retain set information in labels. 3078 3079 bandwidth : float, default 1 3080 Bandwidth used by MeanShift for clustering polygons. 3081 3082 point_size : float, default 100 3083 Size of scatter points. 3084 3085 legend_split : int, default 2 3086 Number of columns in legend. 3087 3088 n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate : parameters passed to UMAP. 3089 3090 threshold : float, default 0.1 3091 Minimum text distance for label adjustment to avoid overlap. 3092 3093 width : int, default 12 3094 Figure width. 3095 3096 height : int, default 10 3097 Figure height. 3098 3099 Returns 3100 ------- 3101 matplotlib.figure.Figure 3102 3103 Raises 3104 ------ 3105 ValueError 3106 If `self.similarity` is None. 3107 3108 Notes 3109 ----- 3110 Builds a precomputed distance matrix combining correlation and euclidean distance, 3111 runs UMAP with metric='precomputed', then overlays cluster hulls (MeanShift + convex hull) 3112 and arrows to indicate nearest neighbors (minimal combined distance). 3113 """ 3114 3115 if self.similarity is None: 3116 raise ValueError( 3117 "Similarity data is missing. Please calculate similarity using self.estimating_similarity." 3118 ) 3119 3120 similarity_data = self.similarity 3121 3122 sim = similarity_data["correlation"] 3123 sim_scaled = (sim - sim.min()) / (sim.max() - sim.min()) 3124 eu_dist = similarity_data["euclidean_dist"] 3125 eu_dist_scaled = (eu_dist - eu_dist.min()) / (eu_dist.max() - eu_dist.min()) 3126 3127 similarity_data["combo_dist"] = (1 - sim_scaled) * eu_dist_scaled 3128 3129 # for nn target 3130 arrow_df = similarity_data.copy() 3131 arrow_df = similarity_data.loc[ 3132 similarity_data.groupby("cell1")["combo_dist"].idxmin() 3133 ] 3134 3135 cells = sorted(set(similarity_data["cell1"]) | set(similarity_data["cell2"])) 3136 combo_matrix = pd.DataFrame(0, index=cells, columns=cells, dtype=float) 3137 3138 for _, row in similarity_data.iterrows(): 3139 combo_matrix.loc[row["cell1"], row["cell2"]] = row["combo_dist"] 3140 combo_matrix.loc[row["cell2"], row["cell1"]] = row["combo_dist"] 3141 3142 umap_model = umap.UMAP( 3143 n_components=2, 3144 metric="precomputed", 3145 n_neighbors=n_neighbors, 3146 min_dist=min_dist, 3147 spread=spread, 3148 set_op_mix_ratio=set_op_mix_ratio, 3149 local_connectivity=local_connectivity, 3150 repulsion_strength=repulsion_strength, 3151 negative_sample_rate=negative_sample_rate, 3152 transform_seed=42, 3153 init="spectral", 3154 random_state=42, 3155 verbose=True, 3156 ) 3157 3158 coords = umap_model.fit_transform(combo_matrix.values) 3159 cell_names = list(combo_matrix.index) 3160 num_cells = len(cell_names) 3161 palette = sns.color_palette("tab20c", num_cells) 3162 3163 if "#" in cell_names[0]: 3164 avsets = set( 3165 [re.sub(".*# ", "", x) for x in similarity_data["cell1"]] 3166 + [re.sub(".*# ", "", x) for x in similarity_data["cell2"]] 3167 ) 3168 num_sets = len(avsets) 3169 color_indices = [i * len(palette) // num_sets for i in range(num_sets)] 3170 color_mapping_sets = { 3171 set_name: palette[i] for i, set_name in zip(color_indices, avsets) 3172 } 3173 color_mapping = { 3174 name: color_mapping_sets[re.sub(".*# ", "", name)] 3175 for i, name in enumerate(cell_names) 3176 } 3177 else: 3178 color_mapping = {name: palette[i] for i, name in enumerate(cell_names)} 3179 3180 meanshift = MeanShift(bandwidth=bandwidth) 3181 labels = meanshift.fit_predict(coords) 3182 3183 fig = plt.figure(figsize=(width, height)) 3184 ax = plt.gca() 3185 3186 unique_labels = set(labels) 3187 cluster_palette = sns.color_palette("hls", len(unique_labels)) 3188 3189 for label in unique_labels: 3190 if label == -1: 3191 continue 3192 cluster_coords = coords[labels == label] 3193 if len(cluster_coords) < 3: 3194 continue 3195 3196 hull = ConvexHull(cluster_coords) 3197 hull_points = cluster_coords[hull.vertices] 3198 3199 centroid = np.mean(hull_points, axis=0) 3200 expanded = hull_points + 0.05 * (hull_points - centroid) 3201 3202 poly = Polygon( 3203 expanded, 3204 closed=True, 3205 facecolor=cluster_palette[label], 3206 edgecolor="none", 3207 alpha=0.2, 3208 zorder=1, 3209 ) 3210 ax.add_patch(poly) 3211 3212 texts = [] 3213 for i, (x, y) in enumerate(coords): 3214 plt.scatter( 3215 x, 3216 y, 3217 s=point_size, 3218 color=color_mapping[cell_names[i]], 3219 edgecolors="black", 3220 linewidths=0.5, 3221 zorder=2, 3222 ) 3223 texts.append( 3224 ax.text( 3225 x, y, str(i), ha="center", va="center", fontsize=8, color="black" 3226 ) 3227 ) 3228 3229 def dist(p1, p2): 3230 return np.sqrt((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) 3231 3232 texts_to_adjust = [] 3233 for i, t1 in enumerate(texts): 3234 for j, t2 in enumerate(texts): 3235 if i >= j: 3236 continue 3237 d = dist( 3238 (t1.get_position()[0], t1.get_position()[1]), 3239 (t2.get_position()[0], t2.get_position()[1]), 3240 ) 3241 if d < threshold: 3242 if t1 not in texts_to_adjust: 3243 texts_to_adjust.append(t1) 3244 if t2 not in texts_to_adjust: 3245 texts_to_adjust.append(t2) 3246 3247 adjust_text( 3248 texts_to_adjust, 3249 expand_text=(1.0, 1.0), 3250 force_text=0.9, 3251 arrowprops=dict(arrowstyle="-", color="gray", lw=0.1), 3252 ax=ax, 3253 ) 3254 3255 for _, row in arrow_df.iterrows(): 3256 try: 3257 idx1 = cell_names.index(row["cell1"]) 3258 idx2 = cell_names.index(row["cell2"]) 3259 except ValueError: 3260 continue 3261 x1, y1 = coords[idx1] 3262 x2, y2 = coords[idx2] 3263 arrow = FancyArrowPatch( 3264 (x1, y1), 3265 (x2, y2), 3266 arrowstyle="->", 3267 color="gray", 3268 linewidth=1.5, 3269 alpha=0.5, 3270 mutation_scale=12, 3271 zorder=0, 3272 ) 3273 ax.add_patch(arrow) 3274 3275 if set_info is False and " # " in cell_names[0]: 3276 3277 legend_elements = [ 3278 Patch( 3279 facecolor=color_mapping[name], 3280 edgecolor="black", 3281 label=f"{i} – {re.sub(' #.*', '', name)}", 3282 ) 3283 for i, name in enumerate(cell_names) 3284 ] 3285 3286 else: 3287 3288 legend_elements = [ 3289 Patch( 3290 facecolor=color_mapping[name], 3291 edgecolor="black", 3292 label=f"{i} – {name}", 3293 ) 3294 for i, name in enumerate(cell_names) 3295 ] 3296 3297 plt.legend( 3298 handles=legend_elements, 3299 title="Cells", 3300 bbox_to_anchor=(1.05, 1), 3301 loc="upper left", 3302 ncol=legend_split, 3303 ) 3304 3305 plt.xlabel("UMAP 1") 3306 plt.ylabel("UMAP 2") 3307 plt.grid(False) 3308 plt.show() 3309 3310 return fig 3311 3312 # subclusters part 3313 3314 def subcluster_prepare(self, features: list, cluster: str): 3315 """ 3316 Prepare a `Clustering` object for subcluster analysis on a selected parent cluster. 3317 3318 Parameters 3319 ---------- 3320 features : list 3321 Features to include for subcluster analysis. 3322 3323 cluster : str 3324 Parent cluster name (used to select matching cells). 3325 3326 Update 3327 ------ 3328 Initializes `self.subclusters_` as a new `Clustering` instance containing the 3329 reduced data for the given cluster and stores `current_features` and `current_cluster`. 3330 """ 3331 3332 dat = self.normalized_data 3333 dat.columns = list(self.input_metadata["cell_names"]) 3334 3335 dat = reduce_data(self.normalized_data, features=features, names=[cluster]) 3336 3337 self.subclusters_ = Clustering(data=dat, metadata=None) 3338 3339 self.subclusters_.current_features = features 3340 self.subclusters_.current_cluster = cluster 3341 3342 def define_subclusters( 3343 self, 3344 umap_num: int = 2, 3345 eps: float = 0.5, 3346 min_samples: int = 10, 3347 n_neighbors: int = 5, 3348 min_dist: float = 0.1, 3349 spread: float = 1.0, 3350 set_op_mix_ratio: float = 1.0, 3351 local_connectivity: int = 1, 3352 repulsion_strength: float = 1.0, 3353 negative_sample_rate: int = 5, 3354 width=8, 3355 height=6, 3356 ): 3357 """ 3358 Compute UMAP and DBSCAN clustering within a previously prepared subcluster dataset. 3359 3360 Parameters 3361 ---------- 3362 umap_num : int, default 2 3363 Number of UMAP dimensions to compute. 3364 3365 eps : float, default 0.5 3366 DBSCAN eps parameter. 3367 3368 min_samples : int, default 10 3369 DBSCAN min_samples parameter. 3370 3371 n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate, width, height : 3372 Additional parameters passed to UMAP / plotting / MeanShift as appropriate. 3373 3374 Update 3375 ------- 3376 Stores cluster labels in `self.subclusters_.subclusters`. 3377 3378 Raises 3379 ------ 3380 RuntimeError 3381 If `self.subclusters_` has not been prepared. 3382 """ 3383 3384 if self.subclusters_ is None: 3385 raise RuntimeError( 3386 "Nothing to return. 'self.subcluster_prepare' was not conducted!" 3387 ) 3388 3389 self.subclusters_.perform_UMAP( 3390 factorize=False, 3391 umap_num=umap_num, 3392 pc_num=0, 3393 harmonized=False, 3394 n_neighbors=n_neighbors, 3395 min_dist=min_dist, 3396 spread=spread, 3397 set_op_mix_ratio=set_op_mix_ratio, 3398 local_connectivity=local_connectivity, 3399 repulsion_strength=repulsion_strength, 3400 negative_sample_rate=negative_sample_rate, 3401 width=width, 3402 height=height, 3403 ) 3404 3405 fig = self.subclusters_.find_clusters_UMAP( 3406 umap_n=umap_num, 3407 eps=eps, 3408 min_samples=min_samples, 3409 width=width, 3410 height=height, 3411 ) 3412 3413 clusters = self.subclusters_.return_clusters(clusters="umap") 3414 3415 self.subclusters_.subclusters = [str(x) for x in list(clusters)] 3416 3417 return fig 3418 3419 def subcluster_features_dot( 3420 self, 3421 colors="viridis", 3422 hclust="complete", 3423 scale=False, 3424 scale_axis: Literal["x", "y"] = "y", 3425 img_width=3, 3426 img_high=5, 3427 label_size=6, 3428 size_scale=70, 3429 y_lab="Genes", 3430 legend_lab="normalized", 3431 bbox_to_anchor_scale: int = 25, 3432 bbox_to_anchor_perc: tuple = (0.91, 0.63), 3433 ): 3434 """ 3435 Create a features-dot visualization for the subclusters (averaged and occurrence). 3436 3437 Parameters 3438 ---------- 3439 colors : str, default 'viridis' 3440 Colormap name passed to `features_dot`. 3441 3442 hclust : str or None 3443 Hierarchical clustering linkage to order rows/columns. 3444 3445 scale: bool, default False 3446 If True, expression data will be scaled (0–1) across the rows (features). 3447 3448 scale_axis : {'x', 'y'}, default 'y' 3449 The axis along which to scale the expression data to the [0, 1] range: 3450 - 'y': Scales across columns (per sample / cell). 3451 - 'x': Scales across rows (per feature / gene). 3452 Only effective if `scale=True`. 3453 3454 img_width, img_high : float 3455 Figure size. 3456 3457 label_size : int 3458 Font size for labels. 3459 3460 size_scale : int 3461 Bubble size scaling. 3462 3463 y_lab : str 3464 X axis label. 3465 3466 legend_lab : str 3467 Colorbar label. 3468 3469 bbox_to_anchor_scale : int, default=25 3470 Vertical scale (percentage) for positioning the colorbar. 3471 3472 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3473 Anchor position for the size legend (percent bubble legend). 3474 3475 Returns 3476 ------- 3477 matplotlib.figure.Figure 3478 3479 Raises 3480 ------ 3481 RuntimeError 3482 If subcluster preparation/definition has not been run. 3483 """ 3484 3485 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3486 raise RuntimeError( 3487 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3488 ) 3489 3490 dat = self.normalized_data 3491 dat.columns = list(self.input_metadata["cell_names"]) 3492 3493 dat = reduce_data( 3494 self.normalized_data, 3495 features=self.subclusters_.current_features, 3496 names=[self.subclusters_.current_cluster], 3497 ) 3498 3499 dat.columns = self.subclusters_.subclusters 3500 3501 avg = average(dat) 3502 occ = occurrence(dat) 3503 3504 scatter = features_dot( 3505 expression_data=avg, 3506 occurence_data=occ, 3507 features=None, 3508 scale=scale, 3509 scale_axis=scale_axis, 3510 metadata_list=None, 3511 colors=colors, 3512 hclust=hclust, 3513 img_width=img_width, 3514 img_high=img_high, 3515 label_size=label_size, 3516 size_scale=size_scale, 3517 y_lab=y_lab, 3518 legend_lab=legend_lab, 3519 bbox_to_anchor_scale=bbox_to_anchor_scale, 3520 bbox_to_anchor_perc=bbox_to_anchor_perc, 3521 ) 3522 3523 return scatter 3524 3525 def subcluster_DEG_dot( 3526 self, 3527 top_n=3, 3528 min_exp=0, 3529 min_pct=0.25, 3530 p_val=0.05, 3531 colors="viridis", 3532 hclust="complete", 3533 scale=False, 3534 scale_axis: Literal["x", "y"] = "y", 3535 img_width=3, 3536 img_high=5, 3537 label_size=6, 3538 size_scale=70, 3539 y_lab="Genes", 3540 legend_lab="normalized", 3541 bbox_to_anchor_scale: int = 25, 3542 bbox_to_anchor_perc: tuple = (0.91, 0.63), 3543 n_proc=10, 3544 ): 3545 """ 3546 Plot top differential features (DEGs) for subclusters as a features-dot. 3547 3548 Parameters 3549 ---------- 3550 top_n : int, default 3 3551 Number of top features per subcluster to show. 3552 3553 min_exp : float, default 0 3554 Minimum expression threshold passed to `statistic`. 3555 3556 min_pct : float, default 0.25 3557 Minimum percent expressed in target group. 3558 3559 p_val: float, default 0.05 3560 Maximum p-value for visualizing features. 3561 3562 n_proc : int, default 10 3563 Parallel jobs used for DEG calculation. 3564 3565 scale: bool, default False 3566 If True, expression_data will be scaled (0–1) across the rows (features). 3567 3568 scale_axis : {'x', 'y'}, default 'y' 3569 The axis along which to scale the expression data to the [0, 1] range: 3570 - 'y': Scales across columns (per sample / cell). 3571 - 'x': Scales across rows (per feature / gene). 3572 Only effective if `scale=True`. 3573 3574 colors : str, default='viridis' 3575 Colormap for expression values. 3576 3577 hclust : str or None, default='complete' 3578 Linkage method for hierarchical clustering. If None, no clustering 3579 is performed. 3580 3581 img_width : int or float, default=8 3582 Width of the plot in inches. 3583 3584 img_high : int or float, default=5 3585 Height of the plot in inches. 3586 3587 label_size : int, default=10 3588 Font size for axis labels and ticks. 3589 3590 size_scale : int or float, default=100 3591 Scaling factor for bubble sizes. 3592 3593 y_lab : str, default='Genes' 3594 Label for the x-axis. 3595 3596 legend_lab : str, default='normalized' 3597 Label for the colorbar legend. 3598 3599 bbox_to_anchor_scale : int, default=25 3600 Vertical scale (percentage) for positioning the colorbar. 3601 3602 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3603 Anchor position for the size legend (percent bubble legend). 3604 3605 Returns 3606 ------- 3607 matplotlib.figure.Figure 3608 3609 Raises 3610 ------ 3611 RuntimeError 3612 If subcluster preparation/definition has not been run. 3613 3614 Notes 3615 ----- 3616 Internally calls `calc_DEG` (or equivalent) to obtain statistics, filters 3617 by p-value and effect-size, selects top features per valid group and plots them. 3618 """ 3619 3620 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3621 raise RuntimeError( 3622 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3623 ) 3624 3625 dat = self.normalized_data 3626 dat.columns = list(self.input_metadata["cell_names"]) 3627 3628 dat = reduce_data( 3629 self.normalized_data, names=[self.subclusters_.current_cluster] 3630 ) 3631 3632 dat.columns = self.subclusters_.subclusters 3633 3634 deg_stats = calc_DEG( 3635 dat, 3636 metadata_list=None, 3637 entities="All", 3638 sets=None, 3639 min_exp=min_exp, 3640 min_pct=min_pct, 3641 n_proc=n_proc, 3642 ) 3643 3644 deg_stats = deg_stats[deg_stats["p_val"] <= p_val] 3645 deg_stats = deg_stats[deg_stats["log(FC)"] > 0] 3646 3647 deg_stats = ( 3648 deg_stats.sort_values( 3649 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 3650 ) 3651 .groupby("valid_group") 3652 .head(top_n) 3653 ) 3654 3655 dat = reduce_data(dat, features=list(set(deg_stats["feature"]))) 3656 3657 avg = average(dat) 3658 occ = occurrence(dat) 3659 3660 scatter = features_dot( 3661 expression_data=avg, 3662 occurence_data=occ, 3663 features=None, 3664 metadata_list=None, 3665 scale_axis=scale_axis, 3666 scale=scale, 3667 colors=colors, 3668 hclust=hclust, 3669 img_width=img_width, 3670 img_high=img_high, 3671 label_size=label_size, 3672 size_scale=size_scale, 3673 y_lab=y_lab, 3674 legend_lab=legend_lab, 3675 bbox_to_anchor_scale=bbox_to_anchor_scale, 3676 bbox_to_anchor_perc=bbox_to_anchor_perc, 3677 ) 3678 3679 return scatter 3680 3681 def accept_subclusters(self): 3682 """ 3683 Commit subcluster labels into the main `input_metadata` by renaming cell names. 3684 3685 The method replaces occurrences of the parent cluster name in `self.input_metadata['cell_names']` 3686 with the expanded names that include subcluster suffixes (via `add_subnames`), 3687 then clears `self.subclusters_`. 3688 3689 Update 3690 ------ 3691 Modifies `self.input_metadata['cell_names']`. 3692 3693 Resets `self.subclusters_` to None. 3694 3695 Raises 3696 ------ 3697 RuntimeError 3698 If `self.subclusters_` is not defined or subclusters were not computed. 3699 """ 3700 3701 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3702 raise RuntimeError( 3703 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3704 ) 3705 3706 new_meta = add_subnames( 3707 list(self.input_metadata["cell_names"]), 3708 parent_name=self.subclusters_.current_cluster, 3709 new_clusters=self.subclusters_.subclusters, 3710 ) 3711 3712 self.input_metadata["cell_names"] = new_meta 3713 3714 self.subclusters_ = None 3715 3716 def dot_plot( 3717 self, 3718 names: list | None = None, 3719 features: list | None = None, 3720 name_slot: str = "cell_names", 3721 scale=True, 3722 scale_axis: Literal["x", "y"] = "y", 3723 colors="viridis", 3724 hclust=None, 3725 img_width=15, 3726 img_high=1, 3727 label_size=10, 3728 size_scale=200, 3729 y_lab="Genes", 3730 legend_lab="log(CPM + 1)", 3731 set_box_size: float | int = 5, 3732 set_box_high: float | int = 5, 3733 bbox_to_anchor_scale=25, 3734 bbox_to_anchor_perc=(0.90, -0.24), 3735 bbox_to_anchor_group=(1.01, 0.4), 3736 ): 3737 """ 3738 Create a dot plot of selected features across samples inside project. 3739 3740 Each point represents a feature-sample pair, where the color encodes the 3741 expression value and the size encodes occurrence or relative abundance. 3742 Optionally, hierarchical clustering can be applied to order rows and columns. 3743 3744 Parameters 3745 ---------- 3746 names : list, str, or None 3747 Names of samples to include. If None, all samples are considered. 3748 3749 features : list, str, or None 3750 Names of features to include. If None, all features are considered. 3751 3752 name_slot : str 3753 Column in metadata to use as sample names. 3754 3755 scale: bool, default False 3756 If True, expression_data will be scaled (0–1) across the rows (features). 3757 3758 scale_axis : {'x', 'y'}, default 'y' 3759 The axis along which to scale the expression data to the [0, 1] range: 3760 - 'y': Scales across columns (per sample / cell). 3761 - 'x': Scales across rows (per feature / gene). 3762 Only effective if `scale=True`. 3763 3764 colors : str, default='viridis' 3765 Colormap for expression values. 3766 3767 hclust : str or None, default='complete' 3768 Linkage method for hierarchical clustering. If None, no clustering 3769 is performed. 3770 3771 img_width : int or float, default=8 3772 Width of the plot in inches. 3773 3774 img_high : int or float, default=5 3775 Height of the plot in inches. 3776 3777 label_size : int, default=10 3778 Font size for axis labels and ticks. 3779 3780 size_scale : int or float, default=100 3781 Scaling factor for bubble sizes. 3782 3783 y_lab : str, default='Genes' 3784 Label for the x-axis. 3785 3786 legend_lab : str, default='log(CPM + 1)' 3787 Label for the colorbar legend. 3788 3789 bbox_to_anchor_scale : int, default=25 3790 Vertical scale (percentage) for positioning the colorbar. 3791 3792 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3793 Anchor position for the size legend (percent bubble legend). 3794 3795 bbox_to_anchor_group : tuple, default=(1.01, 0.4) 3796 Anchor position for the group legend. 3797 3798 Returns 3799 ------- 3800 matplotlib.figure.Figure 3801 The generated dot plot figure. 3802 3803 Notes 3804 ----- 3805 Colors represent expression values normalized to the colormap. 3806 """ 3807 3808 prtd, met = self.get_partial_data( 3809 names=names, features=features, name_slot=name_slot, inc_metadata=True 3810 ) 3811 3812 prtd.columns = prtd.columns + "#" + met["sets"] 3813 3814 prtd_avg = average(prtd) 3815 3816 meta_sets = [re.sub(".*#", "", x) for x in prtd_avg.columns] 3817 3818 prtd_avg.columns = [re.sub("#.*", "", x) for x in prtd_avg.columns] 3819 3820 prtd_occ = occurrence(prtd) 3821 3822 prtd_occ.columns = [re.sub("#.*", "", x) for x in prtd_occ.columns] 3823 3824 fig_scatter = features_dot( 3825 expression_data=prtd_avg, 3826 occurence_data=prtd_occ, 3827 scale=scale, 3828 scale_axis=scale_axis, 3829 features=None, 3830 metadata_list=meta_sets, 3831 colors=colors, 3832 hclust=hclust, 3833 img_width=img_width, 3834 img_high=img_high, 3835 label_size=label_size, 3836 size_scale=size_scale, 3837 y_lab=y_lab, 3838 legend_lab=legend_lab, 3839 set_box_size=set_box_size, 3840 set_box_high=set_box_high, 3841 bbox_to_anchor_scale=bbox_to_anchor_scale, 3842 bbox_to_anchor_perc=bbox_to_anchor_perc, 3843 bbox_to_anchor_group=bbox_to_anchor_group, 3844 ) 3845 3846 return fig_scatter 3847 3848 def data_composition( 3849 self, 3850 features_count: list | None, 3851 name_slot: str = "cell_names", 3852 set_sep: bool = True, 3853 ): 3854 """ 3855 Compute composition of cell types in data set. 3856 3857 This function counts the occurrences of specific cells (e.g., cell types, subtypes) 3858 within metadata entries, calculates their relative percentages, and stores 3859 the results in `self.composition_data`. 3860 3861 Parameters 3862 ---------- 3863 features_count : list or None 3864 List of features (part or full names) to be counted. 3865 If None, all unique elements from the specified `name_slot` metadata field are used. 3866 3867 name_slot : str, default 'cell_names' 3868 Metadata field containing sample identifiers or labels. 3869 3870 set_sep : bool, default True 3871 If True and multiple sets are present in metadata, compute composition 3872 separately for each set. 3873 3874 Update 3875 ------- 3876 Stores results in `self.composition_data` as a pandas DataFrame with: 3877 - 'name': feature name 3878 - 'n': number of occurrences 3879 - 'pct': percentage of occurrences 3880 - 'set' (if applicable): dataset identifier 3881 """ 3882 3883 validated_list = list(self.input_metadata[name_slot]) 3884 sets = list(self.input_metadata["sets"]) 3885 3886 if features_count is None: 3887 features_count = list(set(self.input_metadata[name_slot])) 3888 3889 if set_sep and len(set(sets)) > 1: 3890 3891 final_res = pd.DataFrame() 3892 3893 for s in set(sets): 3894 print(s) 3895 3896 mask = [True if s == x else False for x in sets] 3897 3898 tmp_val_list = np.array(validated_list) 3899 3900 tmp_val_list = list(tmp_val_list[mask]) 3901 3902 res_dict = {"name": [], "n": [], "set": []} 3903 3904 for f in tqdm(features_count): 3905 res_dict["n"].append( 3906 sum(1 for element in tmp_val_list if f in element) 3907 ) 3908 res_dict["name"].append(f) 3909 res_dict["set"].append(s) 3910 res = pd.DataFrame(res_dict) 3911 res["pct"] = res["n"] / sum(res["n"]) * 100 3912 res["pct"] = res["pct"].round(2) 3913 3914 final_res = pd.concat([final_res, res]) 3915 3916 res = final_res.sort_values(["set", "pct"], ascending=[True, False]) 3917 3918 else: 3919 3920 res_dict = {"name": [], "n": []} 3921 3922 for f in tqdm(features_count): 3923 res_dict["n"].append( 3924 sum(1 for element in validated_list if f in element) 3925 ) 3926 res_dict["name"].append(f) 3927 3928 res = pd.DataFrame(res_dict) 3929 res["pct"] = res["n"] / sum(res["n"]) * 100 3930 res["pct"] = res["pct"].round(2) 3931 3932 res = res.sort_values("pct", ascending=False) 3933 3934 self.composition_data = res 3935 3936 def composition_pie( 3937 self, 3938 width=6, 3939 height=6, 3940 font_size=15, 3941 cmap: str = "tab20", 3942 legend_split_col: int = 1, 3943 offset_labels: float | int = 0.5, 3944 legend_bbox: tuple = (1.15, 0.95), 3945 ): 3946 """ 3947 Visualize the composition of cell lineages using pie charts. 3948 3949 Generates pie charts showing the relative proportions of features stored 3950 in `self.composition_data`. If multiple sets are present, a separate 3951 chart is drawn for each set. 3952 3953 Parameters 3954 ---------- 3955 width : int, default 6 3956 Width of the figure. 3957 3958 height : int, default 6 3959 Height of the figure (applied per set if multiple sets are plotted). 3960 3961 font_size : int, default 15 3962 Font size for labels and annotations. 3963 3964 cmap : str, default 'tab20' 3965 Colormap used for pie slices. 3966 3967 legend_split_col : int, default 1 3968 Number of columns in the legend. 3969 3970 offset_labels : float or int, default 0.5 3971 Spacing offset for label placement relative to pie slices. 3972 3973 legend_bbox : tuple, default (1.15, 0.95) 3974 Bounding box anchor position for the legend. 3975 3976 Returns 3977 ------- 3978 matplotlib.figure.Figure 3979 Pie chart visualization of composition data. 3980 """ 3981 3982 df = self.composition_data 3983 3984 if "set" in df.columns and len(set(df["set"])) > 1: 3985 3986 sets = list(set(df["set"])) 3987 fig, axes = plt.subplots(len(sets), 1, figsize=(width, height * len(sets))) 3988 3989 all_wedges = [] 3990 cmap = plt.get_cmap(cmap) 3991 3992 set_nam = len(set(df["name"])) 3993 3994 legend_labels = list(set(df["name"])) 3995 3996 colors = [cmap(i / set_nam) for i in range(set_nam)] 3997 3998 cmap_dict = dict(zip(legend_labels, colors)) 3999 4000 for idx, s in enumerate(sets): 4001 ax = axes[idx] 4002 tmp_df = df[df["set"] == s].reset_index(drop=True) 4003 4004 labels = [f"{row['pct']:.1f}%" for _, row in tmp_df.iterrows()] 4005 4006 wedges, _ = ax.pie( 4007 tmp_df["n"], 4008 startangle=90, 4009 labeldistance=1.05, 4010 colors=[cmap_dict[x] for x in tmp_df["name"]], 4011 wedgeprops={"linewidth": 0.5, "edgecolor": "black"}, 4012 ) 4013 4014 all_wedges.extend(wedges) 4015 4016 kw = dict(arrowprops=dict(arrowstyle="-"), zorder=0, va="center") 4017 n = 0 4018 for i, p in enumerate(wedges): 4019 ang = (p.theta2 - p.theta1) / 2.0 + p.theta1 4020 y = np.sin(np.deg2rad(ang)) 4021 x = np.cos(np.deg2rad(ang)) 4022 horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] 4023 connectionstyle = f"angle,angleA=0,angleB={ang}" 4024 kw["arrowprops"].update({"connectionstyle": connectionstyle}) 4025 if len(labels[i]) > 0: 4026 n += offset_labels 4027 ax.annotate( 4028 labels[i], 4029 xy=(x, y), 4030 xytext=(1.01 * x + (n * x / 4), 1.01 * y + (n * y / 4)), 4031 horizontalalignment=horizontalalignment, 4032 fontsize=font_size, 4033 weight="bold", 4034 **kw, 4035 ) 4036 4037 circle2 = plt.Circle((0, 0), 0.6, color="white", ec="black") 4038 ax.add_artist(circle2) 4039 4040 ax.text( 4041 0, 4042 0, 4043 f"{s}", 4044 ha="center", 4045 va="center", 4046 fontsize=font_size, 4047 weight="bold", 4048 ) 4049 4050 legend_handles = [ 4051 Patch(facecolor=cmap_dict[label], edgecolor="black", label=label) 4052 for label in legend_labels 4053 ] 4054 4055 fig.legend( 4056 handles=legend_handles, 4057 loc="center right", 4058 bbox_to_anchor=legend_bbox, 4059 ncol=legend_split_col, 4060 title="", 4061 ) 4062 4063 plt.tight_layout() 4064 plt.show() 4065 4066 else: 4067 4068 labels = [f"{row['pct']:.1f}%" for _, row in df.iterrows()] 4069 4070 legend_labels = [f"{row['name']}" for _, row in df.iterrows()] 4071 4072 cmap = plt.get_cmap(cmap) 4073 colors = [cmap(i / len(df)) for i in range(len(df))] 4074 4075 fig, ax = plt.subplots( 4076 figsize=(width, height), subplot_kw=dict(aspect="equal") 4077 ) 4078 4079 wedges, _ = ax.pie( 4080 df["n"], 4081 startangle=90, 4082 labeldistance=1.05, 4083 colors=colors, 4084 wedgeprops={"linewidth": 0.5, "edgecolor": "black"}, 4085 ) 4086 4087 kw = dict(arrowprops=dict(arrowstyle="-"), zorder=0, va="center") 4088 n = 0 4089 for i, p in enumerate(wedges): 4090 ang = (p.theta2 - p.theta1) / 2.0 + p.theta1 4091 y = np.sin(np.deg2rad(ang)) 4092 x = np.cos(np.deg2rad(ang)) 4093 horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] 4094 connectionstyle = "angle,angleA=0,angleB={}".format(ang) 4095 kw["arrowprops"].update({"connectionstyle": connectionstyle}) 4096 if len(labels[i]) > 0: 4097 n += offset_labels 4098 4099 ax.annotate( 4100 labels[i], 4101 xy=(x, y), 4102 xytext=(1.01 * x + (n * x / 4), y * 1.01 + (n * y / 4)), 4103 horizontalalignment=horizontalalignment, 4104 fontsize=font_size, 4105 weight="bold", 4106 **kw, 4107 ) 4108 4109 circle2 = plt.Circle((0, 0), 0.6, color="white") 4110 circle2.set_edgecolor("black") 4111 4112 p = plt.gcf() 4113 p.gca().add_artist(circle2) 4114 4115 ax.legend( 4116 wedges, 4117 legend_labels, 4118 title="", 4119 loc="center left", 4120 bbox_to_anchor=legend_bbox, 4121 ncol=legend_split_col, 4122 ) 4123 4124 plt.show() 4125 4126 return fig 4127 4128 def bar_composition( 4129 self, 4130 cmap="tab20b", 4131 width=2, 4132 height=6, 4133 font_size=15, 4134 legend_split_col: int = 1, 4135 legend_bbox: tuple = (1.3, 1), 4136 ): 4137 """ 4138 Visualize the composition of cell lineages using bar plots. 4139 4140 Produces bar plots showing the distribution of features stored in 4141 `self.composition_data`. If multiple sets are present, a separate 4142 bar is drawn for each set. Percentages are annotated alongside the bars. 4143 4144 Parameters 4145 ---------- 4146 cmap : str, default 'tab20b' 4147 Colormap used for stacked bars. 4148 4149 width : int, default 2 4150 Width of each subplot (per set). 4151 4152 height : int, default 6 4153 Height of the figure. 4154 4155 font_size : int, default 15 4156 Font size for labels and annotations. 4157 4158 legend_split_col : int, default 1 4159 Number of columns in the legend. 4160 4161 legend_bbox : tuple, default (1.3, 1) 4162 Bounding box anchor position for the legend. 4163 4164 Returns 4165 ------- 4166 matplotlib.figure.Figure 4167 Stacked bar plot visualization of composition data. 4168 """ 4169 4170 df = self.composition_data 4171 df["num"] = range(1, len(df) + 1) 4172 4173 if "set" in df.columns and len(set(df["set"])) > 1: 4174 4175 sets = list(set(df["set"])) 4176 fig, axes = plt.subplots(1, len(sets), figsize=(width * len(sets), height)) 4177 4178 cmap = plt.get_cmap(cmap) 4179 4180 set_nam = len(set(df["name"])) 4181 4182 legend_labels = list(set(df["name"])) 4183 4184 colors = [cmap(i / set_nam) for i in range(set_nam)] 4185 4186 cmap_dict = dict(zip(legend_labels, colors)) 4187 4188 for idx, s in enumerate(sets): 4189 ax = axes[idx] 4190 4191 tmp_df = df[df["set"] == s].reset_index(drop=True) 4192 4193 values = tmp_df["n"].values 4194 total = sum(values) 4195 values = [v / total * 100 for v in values] 4196 values = [round(v, 2) for v in values] 4197 4198 idx_max = np.argmax(values) 4199 correction = 100 - sum(values) 4200 values[idx_max] += correction 4201 4202 names = tmp_df["name"].values 4203 perc = tmp_df["pct"].values 4204 nums = tmp_df["num"].values 4205 4206 bottom = 0 4207 centers = [] 4208 for name, num, val, color in zip(names, nums, values, colors): 4209 ax.bar(s, val, bottom=bottom, color=cmap_dict[name], label=name) 4210 centers.append(bottom + val / 2) 4211 bottom += val 4212 4213 y_positions = np.linspace(centers[0], centers[-1], len(centers)) 4214 x_text = -0.8 4215 4216 for y_label, y_center, pct, num in zip( 4217 y_positions, centers, perc, nums 4218 ): 4219 ax.annotate( 4220 f"{pct:.1f}%", 4221 xy=(0, y_center), 4222 xycoords="data", 4223 xytext=(x_text, y_label), 4224 textcoords="data", 4225 ha="right", 4226 va="center", 4227 fontsize=font_size, 4228 arrowprops=dict( 4229 arrowstyle="->", 4230 lw=1, 4231 color="black", 4232 connectionstyle="angle3,angleA=0,angleB=90", 4233 ), 4234 ) 4235 4236 ax.set_ylim(0, 100) 4237 ax.set_xlabel(s, fontsize=font_size) 4238 ax.xaxis.label.set_rotation(30) 4239 4240 ax.set_xticks([]) 4241 ax.set_yticks([]) 4242 for spine in ax.spines.values(): 4243 spine.set_visible(False) 4244 4245 legend_handles = [ 4246 Patch(facecolor=cmap_dict[label], edgecolor="black", label=label) 4247 for label in legend_labels 4248 ] 4249 4250 fig.legend( 4251 handles=legend_handles, 4252 loc="center right", 4253 bbox_to_anchor=legend_bbox, 4254 ncol=legend_split_col, 4255 title="", 4256 ) 4257 4258 plt.tight_layout() 4259 plt.show() 4260 4261 else: 4262 4263 cmap = plt.get_cmap(cmap) 4264 4265 colors = [cmap(i / len(df)) for i in range(len(df))] 4266 4267 fig, ax = plt.subplots(figsize=(width, height)) 4268 4269 values = df["n"].values 4270 names = df["name"].values 4271 perc = df["pct"].values 4272 nums = df["num"].values 4273 4274 bottom = 0 4275 centers = [] 4276 for name, num, val, color in zip(names, nums, values, colors): 4277 ax.bar(0, val, bottom=bottom, color=color, label=f"{num}) {name}") 4278 centers.append(bottom + val / 2) 4279 bottom += val 4280 4281 y_positions = np.linspace(centers[0], centers[-1], len(centers)) 4282 x_text = -0.8 4283 4284 for y_label, y_center, pct, num in zip(y_positions, centers, perc, nums): 4285 ax.annotate( 4286 f"{num}) {pct}", 4287 xy=(0, y_center), 4288 xycoords="data", 4289 xytext=(x_text, y_label), 4290 textcoords="data", 4291 ha="right", 4292 va="center", 4293 fontsize=9, 4294 arrowprops=dict( 4295 arrowstyle="->", 4296 lw=1, 4297 color="black", 4298 connectionstyle="angle3,angleA=0,angleB=90", 4299 ), 4300 ) 4301 4302 ax.set_xticks([]) 4303 ax.set_yticks([]) 4304 for spine in ax.spines.values(): 4305 spine.set_visible(False) 4306 4307 ax.legend( 4308 title="Legend", 4309 bbox_to_anchor=legend_bbox, 4310 loc="upper left", 4311 ncol=legend_split_col, 4312 ) 4313 4314 plt.tight_layout() 4315 plt.show() 4316 4317 return fig 4318 4319 def cell_regression( 4320 self, 4321 cell_x: str, 4322 cell_y: str, 4323 set_x: str | None = None, 4324 set_y: str | None = None, 4325 min_log=0.5, 4326 n_max=20, 4327 min_exp=0.1, 4328 min_pct=0.5, 4329 n_proc=10, 4330 min_esm=0.5, 4331 p_val=0.05, 4332 adj=True, 4333 image_width=12, 4334 image_high=7, 4335 ): 4336 """ 4337 Perform linear regression between two selected cell populations and identify 4338 significant and top-ranked differentially expressed genes. 4339 4340 The function compares the expression profiles of two selected cells or cell 4341 populations using linear regression. Each gene is represented as a point in 4342 the regression plot, with expression in `cell_x` on the X-axis and expression 4343 in `cell_y` on the Y-axis. 4344 4345 Differential expression statistics are calculated using `calc_DEG`. Genes are 4346 first filtered according to the minimum absolute log fold change, minimum 4347 absolute effect size (ESM), and statistical significance. Among the 4348 significant genes, up to `n_max` genes are selected within each `valid_group`, 4349 prioritizing genes with the largest standardized absolute regression residuals 4350 and absolute ESM values. 4351 4352 The regression plot uses three point categories: 4353 - gray: genes that are not statistically significant, 4354 - green: statistically significant genes that pass the DEG thresholds, 4355 - blue: top `n_max` genes selected within each `valid_group`. 4356 4357 The selected top genes are additionally annotated with their gene names. 4358 The plot also displays the fitted linear regression line, R-squared, 4359 p-value, and regression equation. 4360 4361 Parameters 4362 ---------- 4363 cell_x : str 4364 Name of the first cell or cell population used as the X-axis. 4365 4366 cell_y : str 4367 Name of the second cell or cell population used as the Y-axis. 4368 4369 set_x : str or None 4370 Dataset or experimental set identifier corresponding to `cell_x`. 4371 Required when multiple datasets contain the same cell name. If None, 4372 the cell is selected only by its name. 4373 4374 set_y : str or None 4375 Dataset or experimental set identifier corresponding to `cell_y`. 4376 Required when multiple datasets contain the same cell name. If None, 4377 the cell is selected only by its name. 4378 4379 min_log : float, default=0.5 4380 Minimum absolute log fold change required for a gene to be considered 4381 for significant DEG selection. 4382 4383 n_max : int, default=20 4384 Maximum number of top genes selected from each `valid_group`. 4385 Genes are ranked primarily by standardized absolute regression residual 4386 and secondarily by absolute Effect Size - Cohen's d. 4387 4388 min_exp : float, default=0.1 4389 Minimum expression threshold passed to `calc_DEG`. 4390 4391 min_pct : float, default=0.5 4392 Minimum fraction of cells in which a gene must be expressed, passed to 4393 `calc_DEG`. 4394 4395 n_proc : int, default=10 4396 Number of parallel processes used by `calc_DEG`. 4397 4398 min_esm : float, default=0.5 4399 Minimum absolute ESM (effect size measure) required for DEG selection. 4400 4401 p_val : float, default=0.05 4402 Maximum allowed p-value for statistical significance. If `adj=True`, 4403 this threshold is applied to the adjusted p-value (`adj_pval`); 4404 otherwise, it is applied to the raw p-value (`p_val`). 4405 4406 adj : bool, default=True 4407 Whether to use the adjusted p-value (`adj_pval`) instead of the raw 4408 p-value (`p_val`) for statistical significance filtering. 4409 4410 image_width : int or float, default=12 4411 Width of the regression plot in inches. 4412 4413 image_high : int or float, default=7 4414 Height of the regression plot in inches. 4415 4416 4417 4418 Returns 4419 ------- 4420 deg_stats : pandas.DataFrame 4421 DataFrame containing the DEG statistics after filtering by absolute 4422 log fold change, absolute ESM, and statistical significance. 4423 4424 fig : matplotlib.figure.Figure 4425 Matplotlib figure containing the regression plot, including expression 4426 points, regression line, regression statistics, and annotations for 4427 the selected top genes. 4428 4429 Raises 4430 ------ 4431 ValueError 4432 If `cell_x` or `cell_y` is not found in the dataset. 4433 4434 ValueError 4435 If multiple columns correspond to the same cell name and the required 4436 dataset identifiers (`set_x` and/or `set_y`) were not provided. 4437 4438 Notes 4439 ----- 4440 * The function calls `self.average()` before performing the analysis. 4441 * Linear regression is calculated using `scipy.stats.linregress`. 4442 * Regression residuals are calculated as the difference between the observed 4443 Y value and the value predicted by the fitted regression model. 4444 * Standardized residuals are used to prioritize genes that deviate most 4445 strongly from the expected linear relationship. 4446 * Significant genes are filtered using absolute log fold change, absolute 4447 ESM, and either raw or adjusted p-value. 4448 * Top genes are selected separately within each `valid_group`. 4449 * The top genes are annotated on the regression plot. 4450 * The returned `deg_stats` contains the significant genes before the final 4451 `n_max` per-group selection. 4452 4453 4454 Examples 4455 -------- 4456 > > > deg_stats, fig = obj.cell_regression( 4457 > > > ... cell_x="NeuronA", 4458 > > > ... cell_y="NeuronB", 4459 > > > ... min_log_min=0.5, 4460 > > > ... min_esm=0.5, 4461 > > > ... p_val=0.05, 4462 > > > ... n_max=20 4463 > > > ... ) 4464 > > > 4465 """ 4466 4467 self.average() 4468 4469 metadata = self.agg_metadata 4470 data = self.agg_normalized_data 4471 data2 = self.normalized_data 4472 metadata2 = self.input_metadata 4473 4474 if set_x is not None and set_y is not None: 4475 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4476 data2.columns = metadata2["cell_names"] + " # " + metadata2["sets"] 4477 cell_x = cell_x + " # " + set_x 4478 cell_y = cell_y + " # " + set_y 4479 4480 deg_stats = calc_DEG( 4481 data2, 4482 metadata_list=None, 4483 entities={ 4484 "cell_x": cell_x + " # " + set_x, 4485 "cell_y": cell_y + " # " + set_y, 4486 }, 4487 sets=None, 4488 min_exp=min_exp, 4489 min_pct=min_pct, 4490 n_proc=n_proc, 4491 ) 4492 4493 else: 4494 data.columns = metadata["cell_names"] 4495 data2.columns = metadata2["cell_names"] 4496 4497 deg_stats = calc_DEG( 4498 data2, 4499 metadata_list=None, 4500 entities={"cell_x": cell_x, "cell_y": cell_y}, 4501 sets=None, 4502 min_exp=min_exp, 4503 min_pct=min_pct, 4504 n_proc=n_proc, 4505 ) 4506 4507 # points 4508 x = data[cell_x] 4509 y = data[cell_y] 4510 4511 # regression 4512 slope, intercept, r_value, p_value, std_err = stats.linregress(x, y) 4513 4514 # prediction 4515 y_pred = slope * x + intercept 4516 4517 # residuals 4518 residual = y - y_pred 4519 4520 # residuals for DEG 4521 deg_stats["residual"] = deg_stats["feature"].map(residual) 4522 deg_stats["residual_abs"] = deg_stats["residual"].abs() 4523 deg_stats["residual_std"] = deg_stats["residual"] / residual.std() 4524 deg_stats["residual_std_abs"] = deg_stats["residual_std"].abs() 4525 deg_stats["esm_abs"] = deg_stats["esm"].abs() 4526 4527 deg_stats = deg_stats[deg_stats["log(FC)"].abs() >= min_log] 4528 4529 deg_stats = deg_stats[deg_stats["esm_abs"] >= min_esm] 4530 4531 if adj: 4532 4533 deg_stats = deg_stats[deg_stats["adj_pval"] <= p_val] 4534 else: 4535 deg_stats = deg_stats[deg_stats["p_val"] <= p_val] 4536 4537 deg_stats_n = ( 4538 deg_stats.sort_values( 4539 ["residual_std_abs", "esm_abs"], ascending=[False, False] 4540 ) 4541 .groupby("valid_group") 4542 .head(n_max) 4543 ) 4544 4545 if not cell_x in data.columns: 4546 raise ValueError("'cell_x' value not in cell names!") 4547 4548 if not cell_y in data.columns: 4549 raise ValueError("'cell_y' value not in cell names!") 4550 4551 if list(data.columns).count(cell_x) > 1: 4552 raise ValueError( 4553 f"'{cell_x}' occurs more than once. If you want to select a specific cell, " 4554 f"please also provide the corresponding 'set_x' and 'set_y' values." 4555 ) 4556 4557 if list(data.columns).count(cell_y) > 1: 4558 raise ValueError( 4559 f"'{cell_y}' occurs more than once. If you want to select a specific cell, " 4560 f"please also provide the corresponding 'set_x' and 'set_y' values." 4561 ) 4562 4563 data["color"] = "gray" 4564 4565 data.loc[data.index.isin(set(deg_stats["feature"])), "color"] = "green" 4566 4567 data.loc[data.index.isin(set(deg_stats_n["feature"])), "color"] = "blue" 4568 4569 color_order = {"gray": 0, "green": 1, "blue": 2} 4570 4571 data["color_order"] = data["color"].map(color_order) 4572 4573 data = data.sort_values("color_order") 4574 4575 fig, ax = plt.subplots(figsize=(image_width, image_high)) 4576 4577 # dots 4578 sns.scatterplot( 4579 x=cell_x, 4580 y=cell_y, 4581 data=data, 4582 hue="color", 4583 palette={"blue": "blue", "green": "green", "gray": "gray"}, 4584 legend=False, 4585 ax=ax, 4586 ) 4587 4588 # regression 4589 sns.regplot(x=cell_x, y=cell_y, data=data, scatter=False, color="red", ax=ax) 4590 4591 legend_elements = [ 4592 Line2D( 4593 [0], 4594 [0], 4595 marker="o", 4596 color="w", 4597 markerfacecolor="gray", 4598 markersize=7, 4599 label="Not significant", 4600 ), 4601 Line2D( 4602 [0], 4603 [0], 4604 marker="o", 4605 color="w", 4606 markerfacecolor="blue", 4607 markersize=7, 4608 label=f"Top {n_max}", 4609 ), 4610 Line2D( 4611 [0], 4612 [0], 4613 marker="o", 4614 color="w", 4615 markerfacecolor="green", 4616 markersize=7, 4617 label="Significant", 4618 ), 4619 Line2D([0], [0], color="red", linewidth=2, label="Linear regression"), 4620 ] 4621 4622 ax.legend(handles=legend_elements, loc="best", frameon=False) 4623 4624 slope, intercept, r_value, p_value, _ = stats.linregress( 4625 data[cell_x], data[cell_y] 4626 ) 4627 4628 equation = "y = {:.2f}x + {:.2f}".format(slope, intercept) 4629 4630 ax.annotate( 4631 "R-squared = {:.2f}\nP-value = {:.2f}\n{}".format( 4632 r_value**2, p_value, equation 4633 ), 4634 xy=(0.05, 0.90), 4635 xycoords="axes fraction", 4636 fontsize=12, 4637 ) 4638 4639 ax.spines["top"].set_visible(False) 4640 ax.spines["right"].set_visible(False) 4641 4642 # oznaczanie genów 4643 def annotate_outliers(x, y, features): 4644 texts = [] 4645 4646 for i, (xi, yi) in enumerate(zip(x, y)): 4647 if data.index[i] in features: 4648 text = ax.text(xi, yi, data.index[i]) 4649 texts.append(text) 4650 4651 return texts 4652 4653 features = list(set(deg_stats_n["feature"])) 4654 4655 texts = annotate_outliers(data[cell_x], data[cell_y], features) 4656 4657 adjust_text(texts, arrowprops=dict(arrowstyle="-", color="gray", alpha=0.5)) 4658 4659 return deg_stats, fig 4660 4661 def cell_genes_distribution( 4662 self, 4663 cells_names: list | None, 4664 cells_sets: list | None = None, 4665 image_width=10, 4666 image_high=7, 4667 ): 4668 """ 4669 Compare the distributions of the number of detected genes per cell across 4670 selected cell populations. 4671 4672 For each selected cell population, the function calculates the number of 4673 detected genes in each cell, defined as the number of genes with an expression 4674 value greater than zero. The resulting distributions are visualized as 4675 overlapping histograms, allowing the gene detection levels of different cell 4676 populations to be compared. 4677 4678 This analysis can be used as a quality-control step before comparing gene 4679 expression profiles or performing differential expression analysis. Similar 4680 distributions of detected genes per cell indicate comparable levels of gene 4681 detection across the selected populations, whereas substantial differences 4682 may indicate differences in sequencing depth, RNA content, or other technical 4683 factors. 4684 4685 Parameters 4686 ---------- 4687 cells_names : list 4688 List of cell population or cluster names to be included in the analysis. 4689 4690 cells_sets : list or None 4691 Optional list of dataset or experimental set identifiers corresponding 4692 element-wise to `cells_names`. This is required when the same cell 4693 population name occurs in multiple datasets and a specific dataset 4694 needs to be selected. 4695 4696 image_width : int or float, default=10 4697 Width of the histogram figure in inches. 4698 4699 image_high : int or float, default=7 4700 Height of the histogram figure in inches. 4701 4702 Returns 4703 ------- 4704 fig : matplotlib.figure.Figure 4705 Matplotlib figure containing overlapping histograms showing the 4706 distribution of the number of detected genes per cell for each selected 4707 cell population. 4708 4709 Raises 4710 ------ 4711 ValueError 4712 If `cells_names` is None or empty. 4713 4714 Notes 4715 ----- 4716 * A gene is considered detected when its expression value is greater than zero. 4717 * The number of detected genes is calculated independently for every cell. 4718 * Each selected cell population is represented by a separate histogram. 4719 * When `cells_sets` is provided and has the same length as `cells_names`, 4720 cell populations are selected using both their population name and dataset 4721 identifier. 4722 * The function operates on `self.normalized_data`. 4723 * The resulting distributions can be used to assess whether selected cell 4724 populations have comparable gene-detection levels before downstream 4725 comparisons. 4726 """ 4727 4728 metadata = self.input_metadata 4729 data = self.normalized_data 4730 4731 if cells_names is None or len(cells_names) == 0: 4732 raise ValueError("No cells selected!") 4733 4734 if cells_sets is not None and len(cells_sets) == len(cells_names): 4735 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4736 full_cells = [] 4737 for n in range(len(cells_names)): 4738 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4739 4740 data = data.loc[:, full_cells] 4741 4742 else: 4743 data.columns = metadata["cell_names"] 4744 data = data.loc[:, cells_names] 4745 4746 fig = plt.figure(figsize=(image_width, image_high)) 4747 4748 for name in data.columns.unique(): 4749 cols = data.columns == name 4750 4751 counts = (data.loc[:, cols] > 0).sum(axis=0) 4752 4753 plt.hist(counts, bins=30, alpha=0.5, label=name) 4754 4755 plt.xlabel("Number of detected genes per cell") 4756 plt.ylabel("Number of cells") 4757 plt.legend() 4758 plt.tight_layout() 4759 4760 return fig 4761 4762 def mm_cell_genes( 4763 self, 4764 cells_names: list | None, 4765 cells_sets: list | None, 4766 non_include_zeros: bool = True, 4767 image_width=7, 4768 image_high=10, 4769 ): 4770 """ 4771 Compare the mean and median gene expression across selected cell populations. 4772 4773 For each selected cell population, the function calculates the mean and median 4774 expression values across all genes and cells. By default, zero-expression 4775 values are excluded from the calculation, allowing the comparison to focus on 4776 genes with detected expression. When `non_include_zeros` is set to False, zero 4777 values are included in the calculation. 4778 4779 The resulting mean and median expression values are visualized as a grouped 4780 bar plot, with separate bars representing the mean and median for each cell 4781 population. This visualization can be used to assess differences in the overall 4782 expression level and distribution between selected cell populations. 4783 4784 Parameters 4785 ---------- 4786 cells_names : list 4787 List of cell population or cluster names to be included in the analysis. 4788 4789 cells_sets : list or None 4790 Optional list of dataset or experimental set identifiers corresponding 4791 element-wise to `cells_names`. When provided and its length matches 4792 `cells_names`, cell populations are selected using both their population 4793 name and dataset identifier. 4794 4795 non_include_zeros : bool, default=True 4796 Whether to exclude zero-expression values before calculating the mean and 4797 median. If True, only non-zero expression values are used. If False, all 4798 expression values, including zeros, are used. 4799 4800 image_width : int or float, default=7 4801 Width of the figure in inches. 4802 4803 image_high : int or float, default=10 4804 Height of the figure in inches. 4805 4806 Returns 4807 ------- 4808 fig : matplotlib.figure.Figure 4809 Matplotlib figure containing grouped bar plots showing the mean and median 4810 expression for each selected cell population. 4811 4812 Raises 4813 ------ 4814 ValueError 4815 If `cells_names` is None or empty. 4816 4817 Notes 4818 ----- 4819 * Expression values are obtained from `self.normalized_data`. 4820 * The mean and median are calculated independently for each selected cell 4821 population. 4822 * When `non_include_zeros=True`, zero-expression values are excluded before 4823 calculating both statistics. 4824 * When `non_include_zeros=False`, zero-expression values contribute to both 4825 the mean and median. 4826 * The analysis uses all genes and cells belonging to each selected population. 4827 * The resulting statistics describe the overall distribution of expression 4828 values and can be used to compare the expression levels of different cell 4829 populations. 4830 """ 4831 4832 metadata = self.input_metadata 4833 data = self.normalized_data 4834 4835 if cells_names is None or len(cells_names) == 0: 4836 raise ValueError("No cells selected!") 4837 4838 if cells_sets is not None and len(cells_sets) == len(cells_names): 4839 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4840 full_cells = [] 4841 for n in range(len(cells_names)): 4842 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4843 4844 data = data.loc[:, full_cells] 4845 4846 else: 4847 data.columns = metadata["cell_names"] 4848 data = data.loc[:, cells_names] 4849 4850 groups = data.columns.unique() 4851 4852 fig = plt.figure(figsize=(image_width, image_high)) 4853 4854 mean_expression = {} 4855 median_expression = {} 4856 4857 for name in groups: 4858 cols = data.columns == name 4859 4860 values = data.loc[:, cols].values.flatten() 4861 4862 if non_include_zeros: 4863 values = values[values != 0] 4864 4865 mean_expression[name] = values.mean() 4866 median_expression[name] = np.median(values) 4867 4868 mean_expression = pd.Series(mean_expression) 4869 median_expression = pd.Series(median_expression) 4870 4871 x = np.arange(len(groups)) 4872 width = 0.35 4873 4874 plt.bar(x - width / 2, mean_expression.values, width, label="Mean") 4875 4876 plt.bar(x + width / 2, median_expression.values, width, label="Median") 4877 4878 plt.xlabel("Cell") 4879 plt.ylabel("Expression") 4880 plt.xticks(x, groups, rotation=45) 4881 plt.legend() 4882 4883 plt.tight_layout() 4884 4885 return fig 4886 4887 def cell_dispersion( 4888 self, 4889 cells_names: list | None, 4890 cells_sets: list | None, 4891 image_width=7, 4892 image_high=10, 4893 ): 4894 """ 4895 Assess the within-cluster dispersion of cell gene expression profiles. 4896 4897 For each selected cell population, the function calculates the centroid of 4898 the population as the mean expression profile across all cells in the cluster. 4899 The Euclidean distance between each individual cell and its corresponding 4900 cluster centroid is then calculated to quantify how much the cell's expression 4901 profile deviates from the average profile of the cluster. 4902 4903 The resulting distributions of distances are visualized using violin plots, 4904 with each violin representing one selected cell population. The distribution 4905 shows the degree of transcriptional heterogeneity within each cluster: 4906 smaller distances indicate cells with more similar expression profiles, 4907 whereas larger distances indicate greater within-cluster variability. 4908 4909 This analysis can be used to assess the internal consistency and transcriptional 4910 heterogeneity of cell populations before comparing their expression profiles 4911 between clusters. 4912 4913 Parameters 4914 ---------- 4915 self : object 4916 self object containing the input metadata and normalized expression 4917 data. 4918 4919 cells_names : list 4920 List of cell population or cluster names to be included in the analysis. 4921 4922 cells_sets : list or None 4923 Optional list of dataset or experimental set identifiers corresponding 4924 element-wise to `cells_names`. When provided and its length matches 4925 `cells_names`, cell populations are selected using both their population 4926 name and dataset identifier. 4927 4928 image_width : int or float, default=7 4929 Width of the figure in inches. 4930 4931 image_high : int or float, default=10 4932 Height of the figure in inches. 4933 4934 Returns 4935 ------- 4936 fig : matplotlib.figure.Figure 4937 Matplotlib figure containing violin plots showing the distribution of 4938 Euclidean distances between individual cells and their respective cluster 4939 centroids. 4940 4941 Raises 4942 ------ 4943 ValueError 4944 If `cells_names` is None or empty. 4945 4946 Notes 4947 ----- 4948 * Expression values are obtained from `self.normalized_data`. 4949 * A cluster centroid is calculated as the mean expression profile across all 4950 cells belonging to that cluster. 4951 * The distance of each cell from the centroid is calculated using Euclidean 4952 distance across all genes. 4953 * Each cell contributes one distance value to the distribution of its cluster. 4954 * Smaller distances indicate lower within-cluster transcriptional dispersion. 4955 * Larger distances indicate greater within-cluster transcriptional dispersion. 4956 * The violin plot includes a box plot showing the median and interquartile 4957 range of the distance distribution. 4958 * Differences in dispersion between clusters can be used to assess their 4959 relative transcriptional homogeneity before downstream comparisons. 4960 """ 4961 4962 metadata = self.input_metadata 4963 data = self.normalized_data 4964 4965 if cells_names is None or len(cells_names) == 0: 4966 raise ValueError("No cells selected!") 4967 4968 if cells_sets is not None and len(cells_sets) == len(cells_names): 4969 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4970 full_cells = [] 4971 for n in range(len(cells_names)): 4972 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4973 4974 data = data.loc[:, full_cells] 4975 4976 else: 4977 data.columns = metadata["cell_names"] 4978 data = data.loc[:, cells_names] 4979 4980 fig = plt.figure(figsize=(image_width, image_high)) 4981 4982 dispersion_long = [] 4983 4984 for cluster in data.columns.unique(): 4985 4986 cells = data.loc[:, data.columns == cluster] 4987 4988 centroid = cells.mean(axis=1) 4989 4990 distances = np.sqrt(((cells.sub(centroid, axis=0)) ** 2).sum(axis=0)) 4991 4992 for cell, distance in distances.items(): 4993 dispersion_long.append( 4994 {"cluster": cluster, "cell": cell, "distance_to_centroid": distance} 4995 ) 4996 4997 dispersion_long = pd.DataFrame(dispersion_long) 4998 4999 sns.violinplot( 5000 data=dispersion_long, x="cluster", y="distance_to_centroid", inner="box" 5001 ) 5002 5003 plt.ylabel("Distance from cluster centroid") 5004 plt.xlabel("Cluster") 5005 plt.xticks(rotation=45) 5006 plt.tight_layout() 5007 5008 return fig
A class COMPsc (Comparison of single-cell data) designed for the integration,
analysis, and visualization of single-cell datasets.
The class supports independent dataset integration, subclustering of existing clusters,
marker detection, and multiple visualization strategies.
The COMPsc class provides methods for:
- Normalizing and filtering single-cell data
- Loading and saving sparse 10x-style datasets
- Computing differential expression and marker genes
- Clustering and subclustering analysis
- Visualizing similarity and spatial relationships
- Aggregating data by cell and set annotations
- Managing metadata and renaming labels
- Plotting gene detection histograms and feature dot plots
Methods
project_dir(path_to_directory, project_list) Scans a directory to create a COMPsc instance mapping project names to their paths.
save_project(name, path=os.getcwd()) Saves the COMPsc object to a pickle file on disk.
load_project(path) Loads a previously saved COMPsc object from a pickle file.
reduce_cols(reg, inc_set=False) Removes columns from data tables where column names contain a specified name or partial substring.
reduce_rows(reg, inc_set=False) Removes rows from data tables where column names contain a specified feature (gene) name.
get_data(set_info=False) Returns normalized data with optional set annotations in column names.
get_metadata() Returns the stored input metadata.
get_partial_data(names=None, features=None, name_slot='cell_names') Return a subset of the data by sample names and/or features.
gene_calculation() Calculates and stores per-cell gene detection counts as a pandas Series.
gene_histograme(bins=100) Plots a histogram of genes detected per cell with an overlaid normal distribution.
gene_threshold(min_n=None, max_n=None) Filters cells based on minimum and/or maximum gene detection thresholds.
load_sparse_from_projects(normalized_data=False) Loads and concatenates sparse 10x-style datasets from project paths into count or normalized data.
rename_names(mapping, slot='cell_names') Renames entries in a specified metadata column using a provided mapping dictionary.
rename_subclusters(mapping) Renames subcluster labels using a provided mapping dictionary.
save_sparse(path_to_save=os.getcwd(), name_slot='cell_names', data_slot='normalized') Exports data as 10x-compatible sparse files (matrix.mtx, barcodes.tsv, genes.tsv).
normalize_data(normalize=True, normalize_factor=100000) Normalizes raw counts to counts-per-specified factor (e.g., CPM-like).
statistic(cells=None, sets=None, min_exp=0.01, min_pct=0.1, n_proc=10) Computes per-feature differential expression statistics (Mann-Whitney U) comparing target vs. rest groups.
calculate_difference_markers(min_exp=0, min_pct=0.25, n_proc=10, force=False) Computes and caches differential markers using the statistic method.
clustering_features(features_list=None, name_slot='cell_names', p_val=0.05, top_n=25, adj_mean=True, beta=0.4) Prepares clustering input by selecting marker features and optionally smoothing cell values.
average() Aggregates normalized data by averaging across (cell_name, set) pairs.
estimating_similarity(method='pearson', p_val=0.05, top_n=25) Computes pairwise correlation and Euclidean distance between aggregated samples.
cell_genes_distribution(cells_names, cells_sets=None, image_width=10, image_high=7) Compares the distributions of detected genes per cell across selected cell populations using overlapping histograms.
mm_cell_genes(cells_names, cells_sets=None, non_include_zeros=True, image_width=7, image_high=10) Calculates and visualizes the mean and median gene expression across selected cell populations as a grouped bar plot.
cell_dispersion(cells_names, cells_sets=None, image_width=7, image_high=10) Assesses within-cluster transcriptional dispersion by calculating distances between individual cells and their cluster centroids, visualized via violin plots.
similarity_plot(split_sets=True, set_info=True, cmap='seismic', width=12, height=10) Visualizes pairwise similarity as a dot plot with correlation as hue and scaled distance as point size.
spatial_similarity(set_info=True, bandwidth=1, n_neighbors=5, min_dist=0.1, legend_split=2, point_size=20, ...) Creates a UMAP-like visualization of similarity relationships with cluster hulls and nearest-neighbor arrows.
subcluster_prepare(features, cluster) Initializes a Clustering object for subcluster analysis on a selected parent cluster.
define_subclusters(umap_num=2, eps=0.5, min_samples=10, bandwidth=1, n_neighbors=5, min_dist=0.1, ...) Performs UMAP and DBSCAN clustering on prepared subcluster data and stores cluster labels.
subcluster_features_dot(colors='viridis', hclust='complete', img_width=3, img_high=5, label_size=6, ...) Visualizes averaged expression and occurrence of features for subclusters as a dot plot.
subcluster_DEG_dot(top_n=3, min_exp=0, min_pct=0.25, p_val=0.05, colors='viridis', ...) Plots top differential features for subclusters as a features-dot visualization.
accept_subclusters() Commits subcluster labels to main metadata by renaming cell names and clears subcluster data.
Raises
ValueError For invalid parameters, mismatched dimensions, or missing metadata.
COMPsc(objects=None)
1155 def __init__( 1156 self, 1157 objects=None, 1158 ): 1159 """ 1160 Initialize the COMPsc class for single-cell data integration and analysis. 1161 1162 Parameters 1163 ---------- 1164 objects : list or None, optional 1165 Optional list of data objects to initialize the instance with. 1166 1167 Attributes 1168 ---------- 1169 -objects : list or None 1170 -input_data : pandas.DataFrame or None 1171 -input_metadata : pandas.DataFrame or None 1172 -normalized_data : pandas.DataFrame or None 1173 -agg_metadata : pandas.DataFrame or None 1174 -agg_normalized_data : pandas.DataFrame or None 1175 -similarity : pandas.DataFrame or None 1176 -var_data : pandas.DataFrame or None 1177 -subclusters_ : instance of Clustering class or None 1178 -cells_calc : pandas.Series or None 1179 -gene_calc : pandas.Series or None 1180 -composition_data : pandas.DataFrame or None 1181 """ 1182 1183 self.objects = objects 1184 """ Stores the input data objects.""" 1185 1186 self.input_data = None 1187 """Raw input data for clustering or integration analysis.""" 1188 1189 self.input_metadata = None 1190 """Metadata associated with the input data.""" 1191 1192 self.normalized_data = None 1193 """Normalized version of the input data.""" 1194 1195 self.agg_metadata = None 1196 '''Aggregated metadata for all sets in object related to "agg_normalized_data"''' 1197 1198 self.agg_normalized_data = None 1199 """Aggregated and normalized data across multiple sets.""" 1200 1201 self.similarity = None 1202 """Similarity data between cells across all samples. and sets""" 1203 1204 self.var_data = None 1205 """DEG analysis results summarizing variance across all samples in the object.""" 1206 1207 self.subclusters_ = None 1208 """Placeholder for information about subclusters analysis; if computed.""" 1209 1210 self.cells_calc = None 1211 """Number of cells detected per sample (grouped by lineage, e.g., cluster or name), reflecting data composition.""" 1212 1213 self.gene_calc = None 1214 """Number of genes detected per sample (cell), reflecting the sequencing depth.""" 1215 1216 self.composition_data = None 1217 """Data describing composition of cells across clusters or sets."""
Initialize the COMPsc class for single-cell data integration and analysis.
Parameters
objects : list or None, optional Optional list of data objects to initialize the instance with.
Attributes
-objects : list or None -input_data : pandas.DataFrame or None -input_metadata : pandas.DataFrame or None -normalized_data : pandas.DataFrame or None -agg_metadata : pandas.DataFrame or None -agg_normalized_data : pandas.DataFrame or None -similarity : pandas.DataFrame or None -var_data : pandas.DataFrame or None -subclusters_ : instance of Clustering class or None -cells_calc : pandas.Series or None -gene_calc : pandas.Series or None -composition_data : pandas.DataFrame or None
cells_calc
Number of cells detected per sample (grouped by lineage, e.g., cluster or name), reflecting data composition.
@classmethod
def
project_dir(cls, path_to_directory, project_list):
1219 @classmethod 1220 def project_dir(cls, path_to_directory, project_list): 1221 """ 1222 Scan a directory and build a COMPsc instance mapping provided project names 1223 to their paths. 1224 1225 Parameters 1226 ---------- 1227 path_to_directory : str 1228 Path containing project subfolders. 1229 1230 project_list : list[str] 1231 List of filenames (folder names) to include in the returned object map. 1232 1233 Returns 1234 ------- 1235 COMPsc 1236 New COMPsc instance with `objects` populated. 1237 1238 Raises 1239 ------ 1240 Exception 1241 A generic exception is caught and a message printed if scanning fails. 1242 1243 Notes 1244 ----- 1245 Function attempts to match entries in `project_list` to directory 1246 names and constructs a simplified object key from the folder name. 1247 """ 1248 try: 1249 objects = {} 1250 for filename in tqdm(os.listdir(path_to_directory)): 1251 for c in project_list: 1252 f = os.path.join(path_to_directory, filename) 1253 if c == filename and os.path.isdir(f): 1254 objects[str(c)] = f 1255 1256 return cls(objects) 1257 1258 except: 1259 print("Something went wrong. Check the function input data and try again!")
Scan a directory and build a COMPsc instance mapping provided project names to their paths.
Parameters
path_to_directory : str Path containing project subfolders.
project_list : list[str] List of filenames (folder names) to include in the returned object map.
Returns
COMPsc
New COMPsc instance with objects populated.
Raises
Exception A generic exception is caught and a message printed if scanning fails.
Notes
Function attempts to match entries in project_list to directory
names and constructs a simplified object key from the folder name.
def
save_project(self, name, path: str = 'C:\\Users\\merag\\Git\\JDti'):
1261 def save_project(self, name, path: str = os.getcwd()): 1262 """ 1263 Save the COMPsc object to disk using pickle. 1264 1265 Parameters 1266 ---------- 1267 name : str 1268 Base filename (without extension) to use when saving. 1269 1270 path : str, default os.getcwd() 1271 Directory in which to save the project file. 1272 1273 Returns 1274 ------- 1275 None 1276 1277 Side Effects 1278 ------------ 1279 - Writes a file `<path>/<name>.jpkl` containing the pickled object. 1280 - Prints a confirmation message with saved path. 1281 """ 1282 1283 full = os.path.join(path, f"{name}.jpkl") 1284 1285 with open(full, "wb") as f: 1286 pickle.dump(self, f) 1287 1288 print(f"Project saved as {full}")
Save the COMPsc object to disk using pickle.
Parameters
name : str Base filename (without extension) to use when saving.
path : str, default os.getcwd() Directory in which to save the project file.
Returns
None
Side Effects
- Writes a file
<path>/<name>.jpklcontaining the pickled object. - Prints a confirmation message with saved path.
@classmethod
def
load_project(cls, path):
1290 @classmethod 1291 def load_project(cls, path): 1292 """ 1293 Load a previously saved COMPsc project from a pickle file. 1294 1295 Parameters 1296 ---------- 1297 path : str 1298 Full path to the pickled project file. 1299 1300 Returns 1301 ------- 1302 COMPsc 1303 The unpickled COMPsc object. 1304 1305 Raises 1306 ------ 1307 FileNotFoundError 1308 If the provided path does not exist. 1309 """ 1310 1311 if not os.path.exists(path): 1312 raise FileNotFoundError("File does not exist!") 1313 with open(path, "rb") as f: 1314 obj = pickle.load(f) 1315 return obj
Load a previously saved COMPsc project from a pickle file.
Parameters
path : str Full path to the pickled project file.
Returns
COMPsc The unpickled COMPsc object.
Raises
FileNotFoundError If the provided path does not exist.
def
reduce_cols( self, reg: str | None = None, full: str | None = None, name_slot: str = 'cell_names', inc_set: bool = False):
1317 def reduce_cols( 1318 self, 1319 reg: str | None = None, 1320 full: str | None = None, 1321 name_slot: str = "cell_names", 1322 inc_set: bool = False, 1323 ): 1324 """ 1325 Remove columns (cells) whose names contain a substring `reg` or 1326 full name `full` from available tables. 1327 1328 Parameters 1329 ---------- 1330 reg : str | None 1331 Substring to search for in column/cell names; matching columns will be removed. 1332 If not None, `full` must be None. 1333 1334 full : str | None 1335 Full name to search for in column/cell names; matching columns will be removed. 1336 If not None, `reg` must be None. 1337 1338 name_slot : str, default 'cell_names' 1339 Column in metadata to use as sample names. 1340 1341 inc_set : bool, default False 1342 If True, column names are interpreted as 'cell_name # set' when matching. 1343 1344 Update 1345 ------------ 1346 Mutates `self.input_data`, `self.normalized_data`, `self.input_metadata`, 1347 `self.agg_normalized_data`, and `self.agg_metadata` (if they exist), 1348 removing columns/rows that match `reg`. 1349 1350 Raises 1351 ------ 1352 Raises ValueError if nothing matches the reduction mask. 1353 """ 1354 1355 if reg is None and full is None: 1356 raise ValueError( 1357 "Both 'reg' and 'full' arguments not provided. Please provide at least one of them!" 1358 ) 1359 1360 if reg is not None and full is not None: 1361 raise ValueError( 1362 "Both 'reg' and 'full' arguments are provided. " 1363 "Please provide only one of them!\n" 1364 "'reg' is used when only part of the name must be detected.\n" 1365 "'full' is used if the full name must be detected." 1366 ) 1367 1368 if reg is not None: 1369 1370 if self.input_data is not None: 1371 1372 if inc_set: 1373 1374 self.input_data.columns = ( 1375 self.input_metadata[name_slot] 1376 + " # " 1377 + self.input_metadata["sets"] 1378 ) 1379 1380 else: 1381 1382 self.input_data.columns = self.input_metadata[name_slot] 1383 1384 mask = [reg.upper() not in x.upper() for x in self.input_data.columns] 1385 1386 if len([y for y in mask if y is False]) == 0: 1387 raise ValueError("Nothing found to reduce") 1388 1389 self.input_data = self.input_data.loc[:, mask] 1390 1391 if self.normalized_data is not None: 1392 1393 if inc_set: 1394 1395 self.normalized_data.columns = ( 1396 self.input_metadata[name_slot] 1397 + " # " 1398 + self.input_metadata["sets"] 1399 ) 1400 1401 else: 1402 1403 self.normalized_data.columns = self.input_metadata[name_slot] 1404 1405 mask = [ 1406 reg.upper() not in x.upper() for x in self.normalized_data.columns 1407 ] 1408 1409 if len([y for y in mask if y is False]) == 0: 1410 raise ValueError("Nothing found to reduce") 1411 1412 self.normalized_data = self.normalized_data.loc[:, mask] 1413 1414 if self.input_metadata is not None: 1415 1416 if inc_set: 1417 1418 self.input_metadata["drop"] = ( 1419 self.input_metadata[name_slot] 1420 + " # " 1421 + self.input_metadata["sets"] 1422 ) 1423 1424 else: 1425 1426 self.input_metadata["drop"] = self.input_metadata[name_slot] 1427 1428 mask = [ 1429 reg.upper() not in x.upper() for x in self.input_metadata["drop"] 1430 ] 1431 1432 if len([y for y in mask if y is False]) == 0: 1433 raise ValueError("Nothing found to reduce") 1434 1435 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 1436 drop=True 1437 ) 1438 1439 self.input_metadata = self.input_metadata.drop( 1440 columns=["drop"], errors="ignore" 1441 ) 1442 1443 if self.agg_normalized_data is not None: 1444 1445 if inc_set: 1446 1447 self.agg_normalized_data.columns = ( 1448 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1449 ) 1450 1451 else: 1452 1453 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1454 1455 mask = [ 1456 reg.upper() not in x.upper() 1457 for x in self.agg_normalized_data.columns 1458 ] 1459 1460 if len([y for y in mask if y is False]) == 0: 1461 raise ValueError("Nothing found to reduce") 1462 1463 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 1464 1465 if self.agg_metadata is not None: 1466 1467 if inc_set: 1468 1469 self.agg_metadata["drop"] = ( 1470 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1471 ) 1472 1473 else: 1474 1475 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1476 1477 mask = [reg.upper() not in x.upper() for x in self.agg_metadata["drop"]] 1478 1479 if len([y for y in mask if y is False]) == 0: 1480 raise ValueError("Nothing found to reduce") 1481 1482 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 1483 drop=True 1484 ) 1485 1486 self.agg_metadata = self.agg_metadata.drop( 1487 columns=["drop"], errors="ignore" 1488 ) 1489 1490 elif full is not None: 1491 1492 if self.input_data is not None: 1493 1494 if inc_set: 1495 1496 self.input_data.columns = ( 1497 self.input_metadata[name_slot] 1498 + " # " 1499 + self.input_metadata["sets"] 1500 ) 1501 1502 if "#" not in full: 1503 1504 self.input_data.columns = self.input_metadata[name_slot] 1505 1506 print( 1507 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1508 "Only the names will be compared, without considering the set information." 1509 ) 1510 1511 else: 1512 1513 self.input_data.columns = self.input_metadata[name_slot] 1514 1515 mask = [full.upper() != x.upper() for x in self.input_data.columns] 1516 1517 if len([y for y in mask if y is False]) == 0: 1518 raise ValueError("Nothing found to reduce") 1519 1520 self.input_data = self.input_data.loc[:, mask] 1521 1522 if self.normalized_data is not None: 1523 1524 if inc_set: 1525 1526 self.normalized_data.columns = ( 1527 self.input_metadata[name_slot] 1528 + " # " 1529 + self.input_metadata["sets"] 1530 ) 1531 1532 if "#" not in full: 1533 1534 self.normalized_data.columns = self.input_metadata[name_slot] 1535 1536 print( 1537 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1538 "Only the names will be compared, without considering the set information." 1539 ) 1540 1541 else: 1542 1543 self.normalized_data.columns = self.input_metadata[name_slot] 1544 1545 mask = [full.upper() != x.upper() for x in self.normalized_data.columns] 1546 1547 if len([y for y in mask if y is False]) == 0: 1548 raise ValueError("Nothing found to reduce") 1549 1550 self.normalized_data = self.normalized_data.loc[:, mask] 1551 1552 if self.input_metadata is not None: 1553 1554 if inc_set: 1555 1556 self.input_metadata["drop"] = ( 1557 self.input_metadata[name_slot] 1558 + " # " 1559 + self.input_metadata["sets"] 1560 ) 1561 1562 if "#" not in full: 1563 1564 self.input_metadata["drop"] = self.input_metadata[name_slot] 1565 1566 print( 1567 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1568 "Only the names will be compared, without considering the set information." 1569 ) 1570 1571 else: 1572 1573 self.input_metadata["drop"] = self.input_metadata[name_slot] 1574 1575 mask = [full.upper() != x.upper() for x in self.input_metadata["drop"]] 1576 1577 if len([y for y in mask if y is False]) == 0: 1578 raise ValueError("Nothing found to reduce") 1579 1580 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 1581 drop=True 1582 ) 1583 1584 self.input_metadata = self.input_metadata.drop( 1585 columns=["drop"], errors="ignore" 1586 ) 1587 1588 if self.agg_normalized_data is not None: 1589 1590 if inc_set: 1591 1592 self.agg_normalized_data.columns = ( 1593 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1594 ) 1595 1596 if "#" not in full: 1597 1598 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1599 1600 print( 1601 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1602 "Only the names will be compared, without considering the set information." 1603 ) 1604 else: 1605 1606 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 1607 1608 mask = [ 1609 full.upper() != x.upper() for x in self.agg_normalized_data.columns 1610 ] 1611 1612 if len([y for y in mask if y is False]) == 0: 1613 raise ValueError("Nothing found to reduce") 1614 1615 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 1616 1617 if self.agg_metadata is not None: 1618 1619 if inc_set: 1620 1621 self.agg_metadata["drop"] = ( 1622 self.agg_metadata[name_slot] + " # " + self.agg_metadata["sets"] 1623 ) 1624 1625 if "#" not in full: 1626 1627 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1628 1629 print( 1630 "Not include the set info (name # set) in the 'full' argument, where 'inc_set' is True" 1631 "Only the names will be compared, without considering the set information." 1632 ) 1633 else: 1634 1635 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 1636 1637 mask = [full.upper() != x.upper() for x in self.agg_metadata["drop"]] 1638 1639 if len([y for y in mask if y is False]) == 0: 1640 raise ValueError("Nothing found to reduce") 1641 1642 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 1643 drop=True 1644 ) 1645 1646 self.agg_metadata = self.agg_metadata.drop( 1647 columns=["drop"], errors="ignore" 1648 ) 1649 1650 self.gene_calculation() 1651 self.cells_calculation()
Remove columns (cells) whose names contain a substring reg or
full name full from available tables.
Parameters
reg : str | None
Substring to search for in column/cell names; matching columns will be removed.
If not None, full must be None.
full : str | None
Full name to search for in column/cell names; matching columns will be removed.
If not None, reg must be None.
name_slot : str, default 'cell_names' Column in metadata to use as sample names.
inc_set : bool, default False If True, column names are interpreted as 'cell_name # set' when matching.
Update
Mutates self.input_data, self.normalized_data, self.input_metadata,
self.agg_normalized_data, and self.agg_metadata (if they exist),
removing columns/rows that match reg.
Raises
Raises ValueError if nothing matches the reduction mask.
def
reduce_rows(self, features_list: list):
1653 def reduce_rows(self, features_list: list): 1654 """ 1655 Remove rows (features) whose names are included in features_list. 1656 1657 Parameters 1658 ---------- 1659 features_list : list 1660 List of features to search for in index/gene names; matching entries will be removed. 1661 1662 Update 1663 ------------ 1664 Mutates `self.input_data`, `self.normalized_data`, `self.input_metadata`, 1665 `self.agg_normalized_data`, and `self.agg_metadata` (if they exist), 1666 removing columns/rows that match `reg`. 1667 1668 Raises 1669 ------ 1670 Prints a message listing features that are not found in the data. 1671 """ 1672 1673 if self.input_data is not None: 1674 1675 res = find_features(self.input_data, features=features_list) 1676 1677 res_list = [x.upper() for x in res["included"]] 1678 1679 mask = [x.upper() not in res_list for x in self.input_data.index] 1680 1681 if len([y for y in mask if y is False]) == 0: 1682 raise ValueError("Nothing found to reduce") 1683 1684 self.input_data = self.input_data.loc[mask, :] 1685 1686 if self.normalized_data is not None: 1687 1688 res = find_features(self.normalized_data, features=features_list) 1689 1690 res_list = [x.upper() for x in res["included"]] 1691 1692 mask = [x.upper() not in res_list for x in self.normalized_data.index] 1693 1694 if len([y for y in mask if y is False]) == 0: 1695 raise ValueError("Nothing found to reduce") 1696 1697 self.normalized_data = self.normalized_data.loc[mask, :] 1698 1699 if self.agg_normalized_data is not None: 1700 1701 res = find_features(self.agg_normalized_data, features=features_list) 1702 1703 res_list = [x.upper() for x in res["included"]] 1704 1705 mask = [x.upper() not in res_list for x in self.agg_normalized_data.index] 1706 1707 if len([y for y in mask if y is False]) == 0: 1708 raise ValueError("Nothing found to reduce") 1709 1710 self.agg_normalized_data = self.agg_normalized_data.loc[mask, :] 1711 1712 if len(res["not_included"]) > 0: 1713 print("\nFeatures not found:") 1714 for i in res["not_included"]: 1715 print(i) 1716 1717 self.gene_calculation() 1718 self.cells_calculation()
Remove rows (features) whose names are included in features_list.
Parameters
features_list : list List of features to search for in index/gene names; matching entries will be removed.
Update
Mutates self.input_data, self.normalized_data, self.input_metadata,
self.agg_normalized_data, and self.agg_metadata (if they exist),
removing columns/rows that match reg.
Raises
Prints a message listing features that are not found in the data.
def
get_data(self, set_info: bool = False):
1720 def get_data(self, set_info: bool = False): 1721 """ 1722 Return normalized data with optional set annotation appended to column names. 1723 1724 Parameters 1725 ---------- 1726 set_info : bool, default False 1727 If True, column names are returned as "cell_name # set"; otherwise 1728 only the `cell_name` is used. 1729 1730 Returns 1731 ------- 1732 pandas.DataFrame 1733 The `self.normalized_data` table with columns renamed according to `set_info`. 1734 1735 Raises 1736 ------ 1737 AttributeError 1738 If `self.normalized_data` or `self.input_metadata` is missing. 1739 """ 1740 1741 to_return = self.normalized_data 1742 1743 if set_info: 1744 to_return.columns = ( 1745 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 1746 ) 1747 else: 1748 to_return.columns = self.input_metadata["cell_names"] 1749 1750 return to_return
Return normalized data with optional set annotation appended to column names.
Parameters
set_info : bool, default False
If True, column names are returned as "cell_name # set"; otherwise
only the cell_name is used.
Returns
pandas.DataFrame
The self.normalized_data table with columns renamed according to set_info.
Raises
AttributeError
If self.normalized_data or self.input_metadata is missing.
def
get_partial_data( self, names: list | str | None = None, features: list | str | None = None, name_slot: str = 'cell_names', inc_metadata: bool = False):
1752 def get_partial_data( 1753 self, 1754 names: list | str | None = None, 1755 features: list | str | None = None, 1756 name_slot: str = "cell_names", 1757 inc_metadata: bool = False, 1758 ): 1759 """ 1760 Return a subset of the data filtered by sample names and/or feature names. 1761 1762 Parameters 1763 ---------- 1764 names : list, str, or None 1765 Names of samples to include. If None, all samples are considered. 1766 1767 features : list, str, or None 1768 Names of features to include. If None, all features are considered. 1769 1770 name_slot : str 1771 Column in metadata to use as sample names. 1772 1773 inc_metadata : bool 1774 If True return tuple (data, metadata) 1775 1776 Returns 1777 ------- 1778 pandas.DataFrame 1779 Subset of the normalized data based on the specified names and features. 1780 """ 1781 1782 data = self.normalized_data.copy() 1783 metadata = self.input_metadata 1784 1785 if name_slot in self.input_metadata.columns: 1786 data.columns = self.input_metadata[name_slot] 1787 else: 1788 raise ValueError("'name_slot' not occured in data!'") 1789 1790 if isinstance(features, str): 1791 features = [features] 1792 elif features is None: 1793 features = [] 1794 1795 if isinstance(names, str): 1796 names = [names] 1797 elif names is None: 1798 names = [] 1799 1800 features = [x.upper() for x in features] 1801 names = [x.upper() for x in names] 1802 1803 columns_names = [x.upper() for x in data.columns] 1804 features_names = [x.upper() for x in data.index] 1805 1806 columns_bool = [True if x in names else False for x in columns_names] 1807 features_bool = [True if x in features else False for x in features_names] 1808 1809 if True not in columns_bool and True not in features_bool: 1810 print("Missing 'names' and/or 'features'. Returning full dataset instead.") 1811 1812 if True in columns_bool: 1813 data = data.loc[:, columns_bool] 1814 metadata = metadata.loc[columns_bool, :] 1815 1816 if True in features_bool: 1817 data = data.loc[features_bool, :] 1818 1819 not_in_features = [y for y in features if y not in features_names] 1820 1821 if len(not_in_features) > 0: 1822 print("\nThe following features were not found in data:") 1823 print("\n".join(not_in_features)) 1824 1825 not_in_names = [y for y in names if y not in columns_names] 1826 1827 if len(not_in_names) > 0: 1828 print("\nThe following names were not found in data:") 1829 print("\n".join(not_in_names)) 1830 1831 if inc_metadata: 1832 return data, metadata 1833 else: 1834 return data
Return a subset of the data filtered by sample names and/or feature names.
Parameters
names : list, str, or None Names of samples to include. If None, all samples are considered.
features : list, str, or None Names of features to include. If None, all features are considered.
name_slot : str Column in metadata to use as sample names.
inc_metadata : bool If True return tuple (data, metadata)
Returns
pandas.DataFrame Subset of the normalized data based on the specified names and features.
def
get_metadata(self):
1836 def get_metadata(self): 1837 """ 1838 Return the stored input metadata. 1839 1840 Returns 1841 ------- 1842 pandas.DataFrame 1843 `self.input_metadata` (may be None if not set). 1844 """ 1845 1846 to_return = self.input_metadata 1847 1848 return to_return
Return the stored input metadata.
Returns
pandas.DataFrame
self.input_metadata (may be None if not set).
def
gene_calculation(self):
1850 def gene_calculation(self): 1851 """ 1852 Calculate and store per-cell counts (e.g., number of detected genes). 1853 1854 The method computes a binary (presence/absence) per cell and sums across 1855 features to produce `self.gene_calc`. 1856 1857 Update 1858 ------ 1859 Sets `self.gene_calc` as a pandas.Series. 1860 1861 Side Effects 1862 ------------ 1863 Uses `self.input_data` when available, otherwise `self.normalized_data`. 1864 """ 1865 1866 if self.input_data is not None: 1867 1868 bin_col = self.input_data.columns.copy() 1869 1870 bin_col = bin_col.where(bin_col <= 0, 1) 1871 1872 sum_data = bin_col.sum(axis=0) 1873 1874 self.gene_calc = sum_data 1875 1876 elif self.normalized_data is not None: 1877 1878 bin_col = self.normalized_data.copy() 1879 1880 bin_col = bin_col.where(bin_col <= 0, 1) 1881 1882 sum_data = bin_col.sum(axis=0) 1883 1884 self.gene_calc = sum_data
Calculate and store per-cell counts (e.g., number of detected genes).
The method computes a binary (presence/absence) per cell and sums across
features to produce self.gene_calc.
Update
Sets self.gene_calc as a pandas.Series.
Side Effects
Uses self.input_data when available, otherwise self.normalized_data.
def
gene_histograme(self, bins=100):
1886 def gene_histograme(self, bins=100): 1887 """ 1888 Plot a histogram of the number of genes detected per cell. 1889 1890 Parameters 1891 ---------- 1892 bins : int, default 100 1893 Number of histogram bins. 1894 1895 Returns 1896 ------- 1897 matplotlib.figure.Figure 1898 Figure containing the histogram of gene contents. 1899 1900 Notes 1901 ----- 1902 Requires `self.gene_calc` to be computed prior to calling. 1903 """ 1904 1905 fig, ax = plt.subplots(figsize=(8, 5)) 1906 1907 _, bin_edges, _ = ax.hist( 1908 self.gene_calc, bins=bins, edgecolor="black", alpha=0.6 1909 ) 1910 1911 mu, sigma = np.mean(self.gene_calc), np.std(self.gene_calc) 1912 1913 x = np.linspace(min(self.gene_calc), max(self.gene_calc), 1000) 1914 y = norm.pdf(x, mu, sigma) 1915 1916 y_scaled = y * len(self.gene_calc) * (bin_edges[1] - bin_edges[0]) 1917 1918 ax.plot( 1919 x, y_scaled, "r-", linewidth=2, label=f"Normal(μ={mu:.2f}, σ={sigma:.2f})" 1920 ) 1921 1922 ax.set_xlabel("Value") 1923 ax.set_ylabel("Count") 1924 ax.set_title("Histogram of genes detected per cell") 1925 1926 ax.set_xticks(np.linspace(min(self.gene_calc), max(self.gene_calc), 20)) 1927 ax.tick_params(axis="x", rotation=90) 1928 1929 ax.legend() 1930 1931 return fig
Plot a histogram of the number of genes detected per cell.
Parameters
bins : int, default 100 Number of histogram bins.
Returns
matplotlib.figure.Figure Figure containing the histogram of gene contents.
Notes
Requires self.gene_calc to be computed prior to calling.
def
gene_threshold(self, min_n: int | None, max_n: int | None):
1933 def gene_threshold(self, min_n: int | None, max_n: int | None): 1934 """ 1935 Filter cells by gene-detection thresholds (min and/or max). 1936 1937 Parameters 1938 ---------- 1939 min_n : int or None 1940 Minimum number of detected genes required to keep a cell. 1941 1942 max_n : int or None 1943 Maximum number of detected genes allowed to keep a cell. 1944 1945 Update 1946 ------- 1947 Filters `self.input_data`, `self.normalized_data`, `self.input_metadata` 1948 (and calls `average()` if `self.agg_normalized_data` exists). 1949 1950 Side Effects 1951 ------------ 1952 Raises ValueError if both bounds are None or if filtering removes all cells. 1953 """ 1954 1955 if min_n is not None and max_n is not None: 1956 mask = (self.gene_calc > min_n) & (self.gene_calc < max_n) 1957 elif min_n is None and max_n is not None: 1958 mask = self.gene_calc < max_n 1959 elif min_n is not None and max_n is None: 1960 mask = self.gene_calc > min_n 1961 else: 1962 raise ValueError("Lack of both min_n and max_n values") 1963 1964 if self.input_data is not None: 1965 1966 if len([y for y in mask if y is False]) == 0: 1967 raise ValueError("Nothing to reduce") 1968 1969 self.input_data = self.input_data.loc[:, mask.values] 1970 1971 if self.normalized_data is not None: 1972 1973 if len([y for y in mask if y is False]) == 0: 1974 raise ValueError("Nothing to reduce") 1975 1976 self.normalized_data = self.normalized_data.loc[:, mask.values] 1977 1978 if self.input_metadata is not None: 1979 1980 if len([y for y in mask if y is False]) == 0: 1981 raise ValueError("Nothing to reduce") 1982 1983 self.input_metadata = self.input_metadata.loc[mask.values, :].reset_index( 1984 drop=True 1985 ) 1986 1987 self.input_metadata = self.input_metadata.drop( 1988 columns=["drop"], errors="ignore" 1989 ) 1990 1991 if self.agg_normalized_data is not None: 1992 self.average() 1993 1994 self.gene_calculation() 1995 self.cells_calculation()
Filter cells by gene-detection thresholds (min and/or max).
Parameters
min_n : int or None Minimum number of detected genes required to keep a cell.
max_n : int or None Maximum number of detected genes allowed to keep a cell.
Update
Filters self.input_data, self.normalized_data, self.input_metadata
(and calls average() if self.agg_normalized_data exists).
Side Effects
Raises ValueError if both bounds are None or if filtering removes all cells.
def
cells_calculation(self, name_slot='cell_names'):
1997 def cells_calculation(self, name_slot="cell_names"): 1998 """ 1999 Calculate number of cells per call name / cluster. 2000 2001 The method computes a binary (presence/absence) per cell name / cluster and sums across 2002 cells. 2003 2004 Parameters 2005 ---------- 2006 name_slot : str, default 'cell_names' 2007 Column in metadata to use as sample names. 2008 2009 Update 2010 ------ 2011 Sets `self.cells_calc` as a pd.DataFrame. 2012 """ 2013 2014 ls = list(self.input_metadata[name_slot]) 2015 2016 df = pd.DataFrame( 2017 { 2018 "cluster": pd.Series(ls).value_counts().index, 2019 "n": pd.Series(ls).value_counts().values, 2020 } 2021 ) 2022 2023 self.cells_calc = df
Calculate number of cells per call name / cluster.
The method computes a binary (presence/absence) per cell name / cluster and sums across cells.
Parameters
name_slot : str, default 'cell_names' Column in metadata to use as sample names.
Update
Sets self.cells_calc as a pd.DataFrame.
def
cell_histograme(self, name_slot: str = 'cell_names'):
2025 def cell_histograme(self, name_slot: str = "cell_names"): 2026 """ 2027 Plot a histogram of the number of cells detected per cell name (cluster). 2028 2029 Parameters 2030 ---------- 2031 name_slot : str, default 'cell_names' 2032 Column in metadata to use as sample names. 2033 2034 Returns 2035 ------- 2036 matplotlib.figure.Figure 2037 Figure containing the histogram of cell contents. 2038 2039 Notes 2040 ----- 2041 Requires `self.cells_calc` to be computed prior to calling. 2042 """ 2043 2044 if name_slot != "cell_names": 2045 self.cells_calculation(name_slot=name_slot) 2046 2047 fig, ax = plt.subplots(figsize=(8, 5)) 2048 2049 _, bin_edges, _ = ax.hist( 2050 list(self.cells_calc["n"]), 2051 bins=len(set(self.cells_calc["cluster"])), 2052 edgecolor="black", 2053 color="orange", 2054 alpha=0.6, 2055 ) 2056 2057 mu, sigma = np.mean(list(self.cells_calc["n"])), np.std( 2058 list(self.cells_calc["n"]) 2059 ) 2060 2061 x = np.linspace( 2062 min(list(self.cells_calc["n"])), max(list(self.cells_calc["n"])), 1000 2063 ) 2064 y = norm.pdf(x, mu, sigma) 2065 2066 y_scaled = y * len(list(self.cells_calc["n"])) * (bin_edges[1] - bin_edges[0]) 2067 2068 ax.plot( 2069 x, y_scaled, "r-", linewidth=2, label=f"Normal(μ={mu:.2f}, σ={sigma:.2f})" 2070 ) 2071 2072 ax.set_xlabel("Value") 2073 ax.set_ylabel("Count") 2074 ax.set_title("Histogram of cells detected per cell name / cluster") 2075 2076 ax.set_xticks( 2077 np.linspace( 2078 min(list(self.cells_calc["n"])), max(list(self.cells_calc["n"])), 20 2079 ) 2080 ) 2081 ax.tick_params(axis="x", rotation=90) 2082 2083 ax.legend() 2084 2085 return fig
Plot a histogram of the number of cells detected per cell name (cluster).
Parameters
name_slot : str, default 'cell_names' Column in metadata to use as sample names.
Returns
matplotlib.figure.Figure Figure containing the histogram of cell contents.
Notes
Requires self.cells_calc to be computed prior to calling.
def
cluster_threshold(self, min_n: int | None, name_slot: str = 'cell_names'):
2087 def cluster_threshold(self, min_n: int | None, name_slot: str = "cell_names"): 2088 """ 2089 Filter cell names / clusters by cell-detection threshold. 2090 2091 Parameters 2092 ---------- 2093 min_n : int or None 2094 Minimum number of detected genes required to keep a cell. 2095 2096 name_slot : str, default 'cell_names' 2097 Column in metadata to use as sample names. 2098 2099 2100 Update 2101 ------- 2102 Filters `self.input_data`, `self.normalized_data`, `self.input_metadata` 2103 (and calls `average()` if `self.agg_normalized_data` exists). 2104 """ 2105 2106 if name_slot != "cell_names": 2107 self.cells_calculation(name_slot=name_slot) 2108 2109 if min_n is not None: 2110 names = self.cells_calc["cluster"][self.cells_calc["n"] < min_n] 2111 else: 2112 raise ValueError("Lack of min_n value") 2113 2114 if len(names) > 0: 2115 2116 if self.input_data is not None: 2117 2118 self.input_data.columns = self.input_metadata[name_slot] 2119 2120 mask = [not any(r in x for r in names) for x in self.input_data.columns] 2121 2122 if len([y for y in mask if y is False]) > 0: 2123 2124 self.input_data = self.input_data.loc[:, mask] 2125 2126 if self.normalized_data is not None: 2127 2128 self.normalized_data.columns = self.input_metadata[name_slot] 2129 2130 mask = [ 2131 not any(r in x for r in names) for x in self.normalized_data.columns 2132 ] 2133 2134 if len([y for y in mask if y is False]) > 0: 2135 2136 self.normalized_data = self.normalized_data.loc[:, mask] 2137 2138 if self.input_metadata is not None: 2139 2140 self.input_metadata["drop"] = self.input_metadata[name_slot] 2141 2142 mask = [ 2143 not any(r in x for r in names) for x in self.input_metadata["drop"] 2144 ] 2145 2146 if len([y for y in mask if y is False]) > 0: 2147 2148 self.input_metadata = self.input_metadata.loc[mask, :].reset_index( 2149 drop=True 2150 ) 2151 2152 self.input_metadata = self.input_metadata.drop( 2153 columns=["drop"], errors="ignore" 2154 ) 2155 2156 if self.agg_normalized_data is not None: 2157 2158 self.agg_normalized_data.columns = self.agg_metadata[name_slot] 2159 2160 mask = [ 2161 not any(r in x for r in names) 2162 for x in self.agg_normalized_data.columns 2163 ] 2164 2165 if len([y for y in mask if y is False]) > 0: 2166 2167 self.agg_normalized_data = self.agg_normalized_data.loc[:, mask] 2168 2169 if self.agg_metadata is not None: 2170 2171 self.agg_metadata["drop"] = self.agg_metadata[name_slot] 2172 2173 mask = [ 2174 not any(r in x for r in names) for x in self.agg_metadata["drop"] 2175 ] 2176 2177 if len([y for y in mask if y is False]) > 0: 2178 2179 self.agg_metadata = self.agg_metadata.loc[mask, :].reset_index( 2180 drop=True 2181 ) 2182 2183 self.agg_metadata = self.agg_metadata.drop( 2184 columns=["drop"], errors="ignore" 2185 ) 2186 2187 self.gene_calculation() 2188 self.cells_calculation()
Filter cell names / clusters by cell-detection threshold.
Parameters
min_n : int or None Minimum number of detected genes required to keep a cell.
name_slot : str, default 'cell_names' Column in metadata to use as sample names.
Update
Filters self.input_data, self.normalized_data, self.input_metadata
(and calls average() if self.agg_normalized_data exists).
def
load_sparse_from_projects(self, normalized_data: bool = False):
2190 def load_sparse_from_projects(self, normalized_data: bool = False): 2191 """ 2192 Load sparse 10x-style datasets from stored project paths, concatenate them, 2193 and populate `input_data` / `normalized_data` and `input_metadata`. 2194 2195 Parameters 2196 ---------- 2197 normalized_data : bool, default False 2198 If True, store concatenated tables in `self.normalized_data`. 2199 If False, store them in `self.input_data` and normalization 2200 is needed using normalize_data() method. 2201 2202 Side Effects 2203 ------------ 2204 - Reads each project using `load_sparse(...)` (expects matrix.mtx, genes.tsv, barcodes.tsv). 2205 - Concatenates all projects column-wise and sets `self.input_metadata`. 2206 - Replaces NaNs with zeros and updates `self.gene_calc`. 2207 """ 2208 2209 obj = self.objects 2210 2211 full_data = pd.DataFrame() 2212 full_metadata = pd.DataFrame() 2213 2214 for ke in obj.keys(): 2215 print(ke) 2216 2217 dt, met = load_sparse(path=obj[ke], name=ke) 2218 2219 full_data = pd.concat([full_data, dt], axis=1) 2220 full_metadata = pd.concat([full_metadata, met], axis=0) 2221 2222 full_data[np.isnan(full_data)] = 0 2223 2224 if normalized_data: 2225 self.normalized_data = full_data 2226 self.input_metadata = full_metadata 2227 else: 2228 2229 self.input_data = full_data 2230 self.input_metadata = full_metadata 2231 2232 self.gene_calculation() 2233 self.cells_calculation()
Load sparse 10x-style datasets from stored project paths, concatenate them,
and populate input_data / normalized_data and input_metadata.
Parameters
normalized_data : bool, default False
If True, store concatenated tables in self.normalized_data.
If False, store them in self.input_data and normalization
is needed using normalize_data() method.
Side Effects
- Reads each project using
load_sparse(...)(expects matrix.mtx, genes.tsv, barcodes.tsv). - Concatenates all projects column-wise and sets
self.input_metadata. - Replaces NaNs with zeros and updates
self.gene_calc.
def
rename_names(self, mapping: dict, slot: str = 'cell_names'):
2235 def rename_names(self, mapping: dict, slot: str = "cell_names"): 2236 """ 2237 Rename entries in `self.input_metadata[slot]` according to a provided mapping. 2238 2239 Parameters 2240 ---------- 2241 mapping : dict 2242 Dictionary with keys 'old_name' and 'new_name', each mapping to a list 2243 of equal length describing replacements. 2244 2245 slot : str, default 'cell_names' 2246 Metadata column to operate on. 2247 2248 Update 2249 ------- 2250 Updates `self.input_metadata[slot]` in-place with renamed values. 2251 2252 Raises 2253 ------ 2254 ValueError 2255 If mapping keys are incorrect, lengths differ, or some 'old_name' values 2256 are not present in the metadata column. 2257 """ 2258 2259 if set(["old_name", "new_name"]) != set(mapping.keys()): 2260 raise ValueError( 2261 "Mapping dictionary must contain keys 'old_name' and 'new_name', " 2262 "each with a list of names to change." 2263 ) 2264 2265 if len(mapping["old_name"]) != len(mapping["new_name"]): 2266 raise ValueError( 2267 "Mapping dictionary lists 'old_name' and 'new_name' " 2268 "must have the same length!" 2269 ) 2270 2271 names_vector = list(self.input_metadata[slot]) 2272 2273 if not all(elem in names_vector for elem in list(mapping["old_name"])): 2274 raise ValueError( 2275 f"Some entries from 'old_name' do not exist in the names of slot {slot}." 2276 ) 2277 2278 replace_dict = dict(zip(mapping["old_name"], mapping["new_name"])) 2279 2280 names_vector_ret = [replace_dict.get(item, item) for item in names_vector] 2281 2282 self.input_metadata[slot] = names_vector_ret
Rename entries in self.input_metadata[slot] according to a provided mapping.
Parameters
mapping : dict Dictionary with keys 'old_name' and 'new_name', each mapping to a list of equal length describing replacements.
slot : str, default 'cell_names' Metadata column to operate on.
Update
Updates self.input_metadata[slot] in-place with renamed values.
Raises
ValueError If mapping keys are incorrect, lengths differ, or some 'old_name' values are not present in the metadata column.
def
rename_subclusters(self, mapping):
2284 def rename_subclusters(self, mapping): 2285 """ 2286 Rename labels stored in `self.subclusters_.subclusters` according to mapping. 2287 2288 Parameters 2289 ---------- 2290 mapping : dict 2291 Mapping with keys 'old_name' and 'new_name' (lists of equal length). 2292 2293 Update 2294 ------- 2295 Updates `self.subclusters_.subclusters` with renamed labels. 2296 2297 Raises 2298 ------ 2299 ValueError 2300 If mapping is invalid or old names are not present. 2301 """ 2302 2303 if set(["old_name", "new_name"]) != set(mapping.keys()): 2304 raise ValueError( 2305 "Mapping dictionary must contain keys 'old_name' and 'new_name', " 2306 "each with a list of names to change." 2307 ) 2308 2309 if len(mapping["old_name"]) != len(mapping["new_name"]): 2310 raise ValueError( 2311 "Mapping dictionary lists 'old_name' and 'new_name' " 2312 "must have the same length!" 2313 ) 2314 2315 names_vector = list(self.subclusters_.subclusters) 2316 2317 if not all(elem in names_vector for elem in list(mapping["old_name"])): 2318 raise ValueError( 2319 "Some entries from 'old_name' do not exist in the subcluster names." 2320 ) 2321 2322 replace_dict = dict(zip(mapping["old_name"], mapping["new_name"])) 2323 2324 names_vector_ret = [replace_dict.get(item, item) for item in names_vector] 2325 2326 self.subclusters_.subclusters = names_vector_ret
Rename labels stored in self.subclusters_.subclusters according to mapping.
Parameters
mapping : dict Mapping with keys 'old_name' and 'new_name' (lists of equal length).
Update
Updates self.subclusters_.subclusters with renamed labels.
Raises
ValueError If mapping is invalid or old names are not present.
def
save_sparse( self, path_to_save: str = 'C:\\Users\\merag\\Git\\JDti', name_slot: str = 'cell_names', data_slot: str = 'normalized'):
2328 def save_sparse( 2329 self, 2330 path_to_save: str = os.getcwd(), 2331 name_slot: str = "cell_names", 2332 data_slot: str = "normalized", 2333 ): 2334 """ 2335 Export data as 10x-compatible sparse files (matrix.mtx, barcodes.tsv, genes.tsv). 2336 2337 Parameters 2338 ---------- 2339 path_to_save : str, default current working directory 2340 Directory where files will be written. 2341 2342 name_slot : str, default 'cell_names' 2343 Metadata column providing cell names for barcodes.tsv. 2344 2345 data_slot : str, default 'normalized' 2346 Either 'normalized' (uses self.normalized_data) or 'count' (uses self.input_data). 2347 2348 Raises 2349 ------ 2350 ValueError 2351 If `data_slot` is not 'normalized' or 'count'. 2352 """ 2353 2354 names = self.input_metadata[name_slot] 2355 2356 if data_slot.lower() == "normalized": 2357 2358 features = list(self.normalized_data.index) 2359 mtx = sparse.csr_matrix(self.normalized_data) 2360 2361 elif data_slot.lower() == "count": 2362 2363 features = list(self.input_data.index) 2364 mtx = sparse.csr_matrix(self.input_data) 2365 2366 else: 2367 raise ValueError("'data_slot' must be included in 'normalized' or 'count'") 2368 2369 os.makedirs(path_to_save, exist_ok=True) 2370 2371 mmwrite(os.path.join(path_to_save, "matrix.mtx"), mtx) 2372 2373 pd.Series(names).to_csv( 2374 os.path.join(path_to_save, "barcodes.tsv"), 2375 index=False, 2376 header=False, 2377 sep="\t", 2378 ) 2379 2380 pd.Series(features).to_csv( 2381 os.path.join(path_to_save, "genes.tsv"), index=False, header=False, sep="\t" 2382 )
Export data as 10x-compatible sparse files (matrix.mtx, barcodes.tsv, genes.tsv).
Parameters
path_to_save : str, default current working directory Directory where files will be written.
name_slot : str, default 'cell_names' Metadata column providing cell names for barcodes.tsv.
data_slot : str, default 'normalized' Either 'normalized' (uses self.normalized_data) or 'count' (uses self.input_data).
Raises
ValueError
If data_slot is not 'normalized' or 'count'.
def
normalize_counts(self, normalize_factor: int = 100000, log_transform: bool = True):
2384 def normalize_counts( 2385 self, normalize_factor: int = 100000, log_transform: bool = True 2386 ): 2387 """ 2388 Normalize raw counts to counts-per-(normalize_factor) 2389 (e.g., CPM, TPM - depending on normalize_factor). 2390 2391 Parameters 2392 ---------- 2393 normalize_factor : int, default 100000 2394 Scaling factor used after dividing by column sums. 2395 2396 log_transform : bool, default True 2397 If True, apply log2(x+1) transformation to normalized values. 2398 2399 Update 2400 ------- 2401 Sets `self.normalized_data` to normalized values (fills NaNs with 0). 2402 2403 Raises 2404 ------ 2405 ValueError 2406 If `self.input_data` is missing (cannot normalize). 2407 """ 2408 if self.input_data is None: 2409 raise ValueError("Input data is missing, cannot normalize.") 2410 2411 sum_col = self.input_data.sum() 2412 self.normalized_data = self.input_data.div(sum_col).fillna(0) * normalize_factor 2413 2414 if log_transform: 2415 # log2(x + 1) to avoid -inf for zeros 2416 self.normalized_data = np.log2(self.normalized_data + 1)
Normalize raw counts to counts-per-(normalize_factor) (e.g., CPM, TPM - depending on normalize_factor).
Parameters
normalize_factor : int, default 100000 Scaling factor used after dividing by column sums.
log_transform : bool, default True If True, apply log2(x+1) transformation to normalized values.
Update
Sets `self.normalized_data` to normalized values (fills NaNs with 0).
Raises
ValueError
If self.input_data is missing (cannot normalize).
def
statistic( self, cells=None, sets=None, min_exp: float = 0.01, min_pct: float = 0.1, n_proc: int = 10):
2418 def statistic( 2419 self, 2420 cells=None, 2421 sets=None, 2422 min_exp: float = 0.01, 2423 min_pct: float = 0.1, 2424 n_proc: int = 10, 2425 ): 2426 """ 2427 Compute per-feature statistics (Mann–Whitney U) comparing target vs rest. 2428 2429 This is a wrapper similar to `calc_DEG` tailored to use `self.normalized_data` 2430 and `self.input_metadata`. It returns per-feature statistics including p-values, 2431 adjusted p-values, means, variances, effect-size measures and fold-changes. 2432 2433 Parameters 2434 ---------- 2435 cells : list, 'All', dict, or None 2436 Defines the target cells or groups for comparison (several modes supported). 2437 2438 sets : 'All', dict, or None 2439 Alternative grouping mode (operate on `self.input_metadata['sets']`). 2440 2441 min_exp : float, default 0.01 2442 Minimum expression threshold used when filtering features. 2443 2444 min_pct : float, default 0.1 2445 Minimum proportion of expressing cells in the target group required to test a feature. 2446 2447 n_proc : int, default 10 2448 Number of parallel jobs to use. 2449 2450 Returns 2451 ------- 2452 pandas.DataFrame or dict 2453 Results DataFrame (or dict containing valid/control cells + DataFrame), 2454 similar to `calc_DEG` interface. 2455 2456 Raises 2457 ------ 2458 ValueError 2459 If neither `cells` nor `sets` is provided, or input metadata mismatch occurs. 2460 2461 Notes 2462 ----- 2463 Multiple modes supported: single-list entities, 'All', pairwise dicts, etc. 2464 """ 2465 2466 offset = 1e-100 2467 2468 def stat_calc(choose, feature_name): 2469 target_values = choose.loc[choose["DEG"] == "target", feature_name] 2470 rest_values = choose.loc[choose["DEG"] == "rest", feature_name] 2471 2472 pct_valid = (target_values > 0).sum() / len(target_values) 2473 pct_rest = (rest_values > 0).sum() / len(rest_values) 2474 2475 avg_valid = np.mean(target_values) 2476 avg_ctrl = np.mean(rest_values) 2477 sd_valid = np.std(target_values, ddof=1) 2478 sd_ctrl = np.std(rest_values, ddof=1) 2479 esm = (avg_valid - avg_ctrl) / np.sqrt(((sd_valid**2 + sd_ctrl**2) / 2)) 2480 2481 if np.sum(target_values) == np.sum(rest_values): 2482 p_val = 1.0 2483 else: 2484 _, p_val = stats.mannwhitneyu( 2485 target_values, rest_values, alternative="two-sided" 2486 ) 2487 2488 return { 2489 "feature": feature_name, 2490 "p_val": p_val, 2491 "pct_valid": pct_valid, 2492 "pct_ctrl": pct_rest, 2493 "avg_valid": avg_valid, 2494 "avg_ctrl": avg_ctrl, 2495 "sd_valid": sd_valid, 2496 "sd_ctrl": sd_ctrl, 2497 "esm": esm, 2498 } 2499 2500 def prepare_and_run_stat(choose, valid_group, min_exp, min_pct, n_proc): 2501 2502 def safe_min_half(series): 2503 filtered = series[(series > ((2**-1074) * 2)) & (series.notna())] 2504 return filtered.min() / 2 if not filtered.empty else 0 2505 2506 tmp_dat = choose[choose["DEG"] == "target"] 2507 tmp_dat = tmp_dat.drop("DEG", axis=1) 2508 2509 counts = (tmp_dat > min_exp).sum(axis=0) 2510 2511 total_count = tmp_dat.shape[0] 2512 2513 info = pd.DataFrame( 2514 {"feature": list(tmp_dat.columns), "pct": list(counts / total_count)} 2515 ) 2516 2517 del tmp_dat 2518 2519 drop_col = info["feature"][info["pct"] <= min_pct] 2520 2521 if len(drop_col) + 1 == len(choose.columns): 2522 drop_col = info["feature"][info["pct"] == 0] 2523 2524 del info 2525 2526 choose = choose.drop(list(drop_col), axis=1) 2527 2528 results = Parallel(n_jobs=n_proc)( 2529 delayed(stat_calc)(choose, feature) 2530 for feature in tqdm(choose.columns[choose.columns != "DEG"]) 2531 ) 2532 2533 df = pd.DataFrame(results) 2534 df = df[(df["avg_valid"] > 0) | (df["avg_ctrl"] > 0)] 2535 2536 df["valid_group"] = valid_group 2537 df.sort_values(by="p_val", inplace=True) 2538 2539 num_tests = len(df) 2540 df["adj_pval"] = np.minimum( 2541 1, (df["p_val"] * num_tests) / np.arange(1, num_tests + 1) 2542 ) 2543 2544 valid_factor = safe_min_half(df["avg_valid"]) 2545 ctrl_factor = safe_min_half(df["avg_ctrl"]) 2546 2547 cv_factor = min(valid_factor, ctrl_factor) 2548 2549 if cv_factor == 0: 2550 cv_factor = max(valid_factor, ctrl_factor) 2551 2552 if not np.isfinite(cv_factor) or cv_factor == 0: 2553 cv_factor += offset 2554 2555 valid = df["avg_valid"].where( 2556 df["avg_valid"] != 0, df["avg_valid"] + cv_factor 2557 ) 2558 ctrl = df["avg_ctrl"].where(df["avg_ctrl"] != 0, df["avg_ctrl"] + cv_factor) 2559 2560 df["FC"] = valid / ctrl 2561 2562 df["log(FC)"] = np.log2(df["FC"]) 2563 df["norm_diff"] = df["avg_valid"] - df["avg_ctrl"] 2564 2565 return df 2566 2567 choose = self.normalized_data.copy().T 2568 2569 final_results = [] 2570 2571 if isinstance(cells, list) and sets is None: 2572 print("\nAnalysis started...\nComparing selected cells to the whole set...") 2573 choose.index = ( 2574 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2575 ) 2576 2577 if "#" not in cells[0]: 2578 choose.index = self.input_metadata["cell_names"] 2579 2580 print( 2581 "Not include the set info (name # set) in the 'cells' list. " 2582 "Only the names will be compared, without considering the set information." 2583 ) 2584 2585 labels = ["target" if idx in cells else "rest" for idx in choose.index] 2586 valid = list( 2587 set(choose.index[[i for i, x in enumerate(labels) if x == "target"]]) 2588 ) 2589 2590 choose["DEG"] = labels 2591 choose = choose[choose["DEG"] != "drop"] 2592 2593 result_df = prepare_and_run_stat( 2594 choose.reset_index(drop=True), 2595 valid_group=valid, 2596 min_exp=min_exp, 2597 min_pct=min_pct, 2598 n_proc=n_proc, 2599 ) 2600 return {"valid_cells": valid, "control_cells": "rest", "DEG": result_df} 2601 2602 elif cells == "All" and sets is None: 2603 print("\nAnalysis started...\nComparing each type of cell to others...") 2604 choose.index = ( 2605 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2606 ) 2607 unique_labels = set(choose.index) 2608 2609 for label in tqdm(unique_labels): 2610 print(f"\nCalculating statistics for {label}") 2611 labels = ["target" if idx == label else "rest" for idx in choose.index] 2612 choose["DEG"] = labels 2613 choose = choose[choose["DEG"] != "drop"] 2614 result_df = prepare_and_run_stat( 2615 choose.copy(), 2616 valid_group=label, 2617 min_exp=min_exp, 2618 min_pct=min_pct, 2619 n_proc=n_proc, 2620 ) 2621 final_results.append(result_df) 2622 2623 return pd.concat(final_results, ignore_index=True) 2624 2625 elif cells is None and sets == "All": 2626 print("\nAnalysis started...\nComparing each set/group to others...") 2627 choose.index = self.input_metadata["sets"] 2628 unique_sets = set(choose.index) 2629 2630 for label in tqdm(unique_sets): 2631 print(f"\nCalculating statistics for {label}") 2632 labels = ["target" if idx == label else "rest" for idx in choose.index] 2633 2634 choose["DEG"] = labels 2635 choose = choose[choose["DEG"] != "drop"] 2636 result_df = prepare_and_run_stat( 2637 choose.copy(), 2638 valid_group=label, 2639 min_exp=min_exp, 2640 min_pct=min_pct, 2641 n_proc=n_proc, 2642 ) 2643 final_results.append(result_df) 2644 2645 return pd.concat(final_results, ignore_index=True) 2646 2647 elif cells is None and isinstance(sets, dict): 2648 print("\nAnalysis started...\nComparing groups...") 2649 2650 choose.index = self.input_metadata["sets"] 2651 2652 group_list = list(sets.keys()) 2653 if len(group_list) != 2: 2654 print("Only pairwise group comparison is supported.") 2655 return None 2656 2657 labels = [ 2658 ( 2659 "target" 2660 if idx in sets[group_list[0]] 2661 else "rest" if idx in sets[group_list[1]] else "drop" 2662 ) 2663 for idx in choose.index 2664 ] 2665 choose["DEG"] = labels 2666 choose = choose[choose["DEG"] != "drop"] 2667 2668 result_df = prepare_and_run_stat( 2669 choose.reset_index(drop=True), 2670 valid_group=group_list[0], 2671 min_exp=min_exp, 2672 min_pct=min_pct, 2673 n_proc=n_proc, 2674 ) 2675 return result_df 2676 2677 elif isinstance(cells, dict) and sets is None: 2678 print("\nAnalysis started...\nComparing groups...") 2679 choose.index = ( 2680 self.input_metadata["cell_names"] + " # " + self.input_metadata["sets"] 2681 ) 2682 2683 if "#" not in cells[list(cells.keys())[0]][0]: 2684 choose.index = self.input_metadata["cell_names"] 2685 2686 print( 2687 "Not include the set info (name # set) in the 'cells' dict. " 2688 "Only the names will be compared, without considering the set information." 2689 ) 2690 2691 group_list = list(cells.keys()) 2692 if len(group_list) != 2: 2693 print("Only pairwise group comparison is supported.") 2694 return None 2695 2696 labels = [ 2697 ( 2698 "target" 2699 if idx in cells[group_list[0]] 2700 else "rest" if idx in cells[group_list[1]] else "drop" 2701 ) 2702 for idx in choose.index 2703 ] 2704 2705 choose["DEG"] = labels 2706 choose = choose[choose["DEG"] != "drop"] 2707 2708 result_df = prepare_and_run_stat( 2709 choose.reset_index(drop=True), 2710 valid_group=group_list[0], 2711 min_exp=min_exp, 2712 min_pct=min_pct, 2713 n_proc=n_proc, 2714 ) 2715 2716 return result_df.reset_index(drop=True) 2717 2718 else: 2719 raise ValueError( 2720 "You must specify either 'cells' or 'sets' (or both). None were provided, which is not allowed for this analysis." 2721 )
Compute per-feature statistics (Mann–Whitney U) comparing target vs rest.
This is a wrapper similar to calc_DEG tailored to use self.normalized_data
and self.input_metadata. It returns per-feature statistics including p-values,
adjusted p-values, means, variances, effect-size measures and fold-changes.
Parameters
cells : list, 'All', dict, or None Defines the target cells or groups for comparison (several modes supported).
sets : 'All', dict, or None
Alternative grouping mode (operate on self.input_metadata['sets']).
min_exp : float, default 0.01 Minimum expression threshold used when filtering features.
min_pct : float, default 0.1 Minimum proportion of expressing cells in the target group required to test a feature.
n_proc : int, default 10 Number of parallel jobs to use.
Returns
pandas.DataFrame or dict
Results DataFrame (or dict containing valid/control cells + DataFrame),
similar to calc_DEG interface.
Raises
ValueError
If neither cells nor sets is provided, or input metadata mismatch occurs.
Notes
Multiple modes supported: single-list entities, 'All', pairwise dicts, etc.
def
calculate_difference_markers(self, min_exp=0, min_pct=0.25, n_proc=10, force: bool = False):
2723 def calculate_difference_markers( 2724 self, min_exp=0, min_pct=0.25, n_proc=10, force: bool = False 2725 ): 2726 """ 2727 Compute differential markers (var_data) if not already present. 2728 2729 Parameters 2730 ---------- 2731 min_exp : float, default 0 2732 Minimum expression threshold passed to `statistic`. 2733 2734 min_pct : float, default 0.25 2735 Minimum percent expressed in target group. 2736 2737 n_proc : int, default 10 2738 Parallel jobs. 2739 2740 force : bool, default False 2741 If True, recompute even if `self.var_data` is present. 2742 2743 Update 2744 ------- 2745 Sets `self.var_data` to the result of `self.statistic(...)`. 2746 2747 Raise 2748 ------ 2749 ValueError if already computed and `force` is False. 2750 """ 2751 2752 if self.var_data is None or force: 2753 2754 self.var_data = self.statistic( 2755 cells="All", sets=None, min_exp=min_exp, min_pct=min_pct, n_proc=n_proc 2756 ) 2757 2758 else: 2759 raise ValueError( 2760 "self.calculate_difference_markers() has already been executed. " 2761 "The results are stored in self.var. " 2762 "If you want to recalculate with different statistics, please rerun the method with force=True." 2763 )
Compute differential markers (var_data) if not already present.
Parameters
min_exp : float, default 0
Minimum expression threshold passed to statistic.
min_pct : float, default 0.25 Minimum percent expressed in target group.
n_proc : int, default 10 Parallel jobs.
force : bool, default False
If True, recompute even if self.var_data is present.
Update
Sets self.var_data to the result of self.statistic(...).
Raise
ValueError if already computed and force is False.
def
clustering_features( self, features_list: list | None, name_slot: str = 'cell_names', p_val: float = 0.05, top_n: int = 25, adj_mean: bool = True, beta: float = 0.2):
2765 def clustering_features( 2766 self, 2767 features_list: list | None, 2768 name_slot: str = "cell_names", 2769 p_val: float = 0.05, 2770 top_n: int = 25, 2771 adj_mean: bool = True, 2772 beta: float = 0.2, 2773 ): 2774 """ 2775 Prepare clustering input by selecting marker features and optionally smoothing cell values 2776 toward group means. 2777 2778 Parameters 2779 ---------- 2780 features_list : list or None 2781 If provided, use this list of features. If None, features are selected 2782 from `self.var_data` (adj_pval <= p_val, positive logFC) picking `top_n` per group. 2783 2784 name_slot : str, default 'cell_names' 2785 Metadata column used for naming. 2786 2787 p_val : float, default 0.05 2788 Adjusted p-value cutoff when selecting features automatically. 2789 2790 top_n : int, default 25 2791 Number of top features per valid group to keep if `features_list` is None. 2792 2793 adj_mean : bool, default True 2794 If True, adjust cell values toward group means using `beta`. 2795 2796 beta : float, default 0.2 2797 Adjustment strength toward group mean. 2798 2799 Update 2800 ------ 2801 Sets `self.clustering_data` and `self.clustering_metadata` to the selected subset, 2802 ready for PCA/UMAP/clustering. 2803 """ 2804 2805 if features_list is None or len(features_list) == 0: 2806 2807 if self.var_data is None: 2808 raise ValueError( 2809 "Lack of 'self.var_data'. Use self.calculate_difference_markers() method first." 2810 ) 2811 2812 df_tmp = self.var_data[self.var_data["adj_pval"] <= p_val] 2813 df_tmp = df_tmp[df_tmp["log(FC)"] > 0] 2814 df_tmp = ( 2815 df_tmp.sort_values( 2816 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 2817 ) 2818 .groupby("valid_group") 2819 .head(top_n) 2820 ) 2821 2822 feaures_list = list(set(df_tmp["feature"])) 2823 2824 data = self.get_partial_data( 2825 names=None, features=feaures_list, name_slot=name_slot 2826 ) 2827 data_avg = average(data) 2828 2829 if adj_mean: 2830 data = adjust_cells_to_group_mean(data=data, data_avg=data_avg, beta=beta) 2831 2832 self.clustering_data = data 2833 2834 self.clustering_metadata = self.input_metadata
Prepare clustering input by selecting marker features and optionally smoothing cell values toward group means.
Parameters
features_list : list or None
If provided, use this list of features. If None, features are selected
from self.var_data (adj_pval <= p_val, positive logFC) picking top_n per group.
name_slot : str, default 'cell_names' Metadata column used for naming.
p_val : float, default 0.05 Adjusted p-value cutoff when selecting features automatically.
top_n : int, default 25
Number of top features per valid group to keep if features_list is None.
adj_mean : bool, default True
If True, adjust cell values toward group means using beta.
beta : float, default 0.2 Adjustment strength toward group mean.
Update
Sets self.clustering_data and self.clustering_metadata to the selected subset,
ready for PCA/UMAP/clustering.
def
average(self):
2836 def average(self): 2837 """ 2838 Aggregate normalized data by (cell_name, set) pairs computing the mean per group. 2839 2840 The method constructs new column names as "cell_name # set", averages columns 2841 sharing identical labels, and populates `self.agg_normalized_data` and `self.agg_metadata`. 2842 2843 Update 2844 ------ 2845 Sets `self.agg_normalized_data` (features x aggregated samples) and 2846 `self.agg_metadata` (DataFrame with 'cell_names' and 'sets'). 2847 """ 2848 2849 wide_data = self.normalized_data 2850 2851 wide_metadata = self.input_metadata 2852 2853 new_names = wide_metadata["cell_names"] + " # " + wide_metadata["sets"] 2854 2855 wide_data.columns = list(new_names) 2856 2857 aggregated_df = wide_data.T.groupby(level=0).mean().T 2858 2859 sets = [re.sub(".*# ", "", x) for x in aggregated_df.columns] 2860 names = [re.sub(" #.*", "", x) for x in aggregated_df.columns] 2861 2862 aggregated_df.columns = names 2863 aggregated_metadata = pd.DataFrame({"cell_names": names, "sets": sets}) 2864 2865 self.agg_metadata = aggregated_metadata 2866 self.agg_normalized_data = aggregated_df
Aggregate normalized data by (cell_name, set) pairs computing the mean per group.
The method constructs new column names as "cell_name # set", averages columns
sharing identical labels, and populates self.agg_normalized_data and self.agg_metadata.
Update
Sets self.agg_normalized_data (features x aggregated samples) and
self.agg_metadata (DataFrame with 'cell_names' and 'sets').
def
estimating_similarity(self, method='pearson', p_val: float = 0.05, top_n: int = 25):
2868 def estimating_similarity( 2869 self, method="pearson", p_val: float = 0.05, top_n: int = 25 2870 ): 2871 """ 2872 Estimate pairwise similarity and Euclidean distance between aggregated samples. 2873 2874 Parameters 2875 ---------- 2876 method : str, default 'pearson' 2877 Correlation method to use (passed to pandas.DataFrame.corr()). 2878 2879 p_val : float, default 0.05 2880 Adjusted p-value cutoff used to select marker features from `self.var_data`. 2881 2882 top_n : int, default 25 2883 Number of top features per valid group to include. 2884 2885 Update 2886 ------- 2887 Computes a combined table with per-pair correlation and euclidean distance 2888 and stores it in `self.similarity`. 2889 """ 2890 2891 if self.var_data is None: 2892 raise ValueError( 2893 "Lack of 'self.var_data'. Use self.calculate_difference_markers() method first." 2894 ) 2895 2896 if self.agg_normalized_data is None: 2897 self.average() 2898 2899 metadata = self.agg_metadata 2900 data = self.agg_normalized_data 2901 2902 df_tmp = self.var_data[self.var_data["adj_pval"] <= p_val] 2903 df_tmp = df_tmp[df_tmp["log(FC)"] > 0] 2904 df_tmp = ( 2905 df_tmp.sort_values( 2906 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 2907 ) 2908 .groupby("valid_group") 2909 .head(top_n) 2910 ) 2911 2912 data = data.loc[list(set(df_tmp["feature"]))] 2913 2914 if len(set(metadata["sets"])) > 1: 2915 data.columns = data.columns + " # " + [x for x in metadata["sets"]] 2916 else: 2917 data = data.copy() 2918 2919 scaler = StandardScaler() 2920 2921 scaled_data = scaler.fit_transform(data) 2922 2923 scaled_df = pd.DataFrame(scaled_data, columns=data.columns) 2924 2925 cor = scaled_df.corr(method=method) 2926 cor_df = cor.stack().reset_index() 2927 cor_df.columns = ["cell1", "cell2", "correlation"] 2928 2929 distances = pdist(scaled_df.T, metric="euclidean") 2930 dist_mat = pd.DataFrame( 2931 squareform(distances), index=scaled_df.columns, columns=scaled_df.columns 2932 ) 2933 dist_df = dist_mat.stack().reset_index() 2934 dist_df.columns = ["cell1", "cell2", "euclidean_dist"] 2935 2936 full = pd.merge(cor_df, dist_df, on=["cell1", "cell2"]) 2937 2938 full = full[full["cell1"] != full["cell2"]] 2939 full = full.reset_index(drop=True) 2940 2941 self.similarity = full
Estimate pairwise similarity and Euclidean distance between aggregated samples.
Parameters
method : str, default 'pearson' Correlation method to use (passed to pandas.DataFrame.corr()).
p_val : float, default 0.05
Adjusted p-value cutoff used to select marker features from self.var_data.
top_n : int, default 25 Number of top features per valid group to include.
Update
Computes a combined table with per-pair correlation and euclidean distance
and stores it in self.similarity.
def
similarity_plot( self, split_sets=True, set_info: bool = True, cmap='seismic', width=12, height=10):
2943 def similarity_plot( 2944 self, 2945 split_sets=True, 2946 set_info: bool = True, 2947 cmap="seismic", 2948 width=12, 2949 height=10, 2950 ): 2951 """ 2952 Visualize similarity as a dot plot. 2953 2954 Parameters 2955 ---------- 2956 split_sets : bool, default True 2957 If True and set information is present, split plotting area roughly into two halves to visualize cross-set pairs. 2958 2959 set_info : bool, default True 2960 If True, keep the ' # set' annotation in labels; otherwise strip it. 2961 2962 cmap : str, default 'seismic' 2963 Color map for correlation (hue). 2964 2965 width : int, default 12 2966 Figure width. 2967 2968 height : int, default 10 2969 Figure height. 2970 2971 Returns 2972 ------- 2973 matplotlib.figure.Figure 2974 2975 Raises 2976 ------ 2977 ValueError 2978 If `self.similarity` is None. 2979 2980 Notes 2981 ----- 2982 The function filters pairs by z-scored euclidean distance > 0 to focus on closer pairs. 2983 """ 2984 2985 if self.similarity is None: 2986 raise ValueError( 2987 "Similarity data is missing. Please calculate similarity using self.estimating_similarity." 2988 ) 2989 2990 similarity_data = self.similarity 2991 2992 if " # " in similarity_data["cell1"][0]: 2993 similarity_data["set1"] = [ 2994 re.sub(".*# ", "", x) for x in similarity_data["cell1"] 2995 ] 2996 similarity_data["set2"] = [ 2997 re.sub(".*# ", "", x) for x in similarity_data["cell2"] 2998 ] 2999 3000 if split_sets and " # " in similarity_data["cell1"][0]: 3001 sets = list( 3002 set(list(similarity_data["set1"]) + list(similarity_data["set2"])) 3003 ) 3004 3005 mm = math.ceil(len(sets) / 2) 3006 3007 x_s = sets[0:mm] 3008 y_s = sets[mm : len(sets)] 3009 3010 similarity_data = similarity_data[similarity_data["set1"].isin(x_s)] 3011 similarity_data = similarity_data[similarity_data["set2"].isin(y_s)] 3012 3013 similarity_data = similarity_data.sort_values(["set1", "set2"]) 3014 3015 if set_info is False and " # " in similarity_data["cell1"][0]: 3016 similarity_data["cell1"] = [ 3017 re.sub(" #.*", "", x) for x in similarity_data["cell1"] 3018 ] 3019 similarity_data["cell2"] = [ 3020 re.sub(" #.*", "", x) for x in similarity_data["cell2"] 3021 ] 3022 3023 similarity_data["-euclidean_zscore"] = -zscore( 3024 similarity_data["euclidean_dist"] 3025 ) 3026 3027 similarity_data = similarity_data[similarity_data["-euclidean_zscore"] > 0] 3028 3029 fig = plt.figure(figsize=(width, height)) 3030 sns.scatterplot( 3031 data=similarity_data, 3032 x="cell1", 3033 y="cell2", 3034 hue="correlation", 3035 size="-euclidean_zscore", 3036 sizes=(1, 100), 3037 palette=cmap, 3038 alpha=1, 3039 edgecolor="black", 3040 ) 3041 3042 plt.xticks(rotation=90) 3043 plt.yticks(rotation=0) 3044 plt.xlabel("Cell 1") 3045 plt.ylabel("Cell 2") 3046 plt.legend(bbox_to_anchor=(1.05, 1), loc="upper left") 3047 3048 plt.grid(True, alpha=0.6) 3049 3050 plt.tight_layout() 3051 3052 return fig
Visualize similarity as a dot plot.
Parameters
split_sets : bool, default True If True and set information is present, split plotting area roughly into two halves to visualize cross-set pairs.
set_info : bool, default True If True, keep the ' # set' annotation in labels; otherwise strip it.
cmap : str, default 'seismic' Color map for correlation (hue).
width : int, default 12 Figure width.
height : int, default 10 Figure height.
Returns
matplotlib.figure.Figure
Raises
ValueError
If self.similarity is None.
Notes
The function filters pairs by z-scored euclidean distance > 0 to focus on closer pairs.
def
spatial_similarity( self, set_info: bool = True, bandwidth=1, n_neighbors=5, min_dist=0.1, legend_split=2, point_size=100, spread=1.0, set_op_mix_ratio=1.0, local_connectivity=1, repulsion_strength=1.0, negative_sample_rate=5, threshold=0.1, width=12, height=10):
3054 def spatial_similarity( 3055 self, 3056 set_info: bool = True, 3057 bandwidth=1, 3058 n_neighbors=5, 3059 min_dist=0.1, 3060 legend_split=2, 3061 point_size=100, 3062 spread=1.0, 3063 set_op_mix_ratio=1.0, 3064 local_connectivity=1, 3065 repulsion_strength=1.0, 3066 negative_sample_rate=5, 3067 threshold=0.1, 3068 width=12, 3069 height=10, 3070 ): 3071 """ 3072 Create a spatial UMAP-like visualization of similarity relationships between samples. 3073 3074 Parameters 3075 ---------- 3076 set_info : bool, default True 3077 If True, retain set information in labels. 3078 3079 bandwidth : float, default 1 3080 Bandwidth used by MeanShift for clustering polygons. 3081 3082 point_size : float, default 100 3083 Size of scatter points. 3084 3085 legend_split : int, default 2 3086 Number of columns in legend. 3087 3088 n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate : parameters passed to UMAP. 3089 3090 threshold : float, default 0.1 3091 Minimum text distance for label adjustment to avoid overlap. 3092 3093 width : int, default 12 3094 Figure width. 3095 3096 height : int, default 10 3097 Figure height. 3098 3099 Returns 3100 ------- 3101 matplotlib.figure.Figure 3102 3103 Raises 3104 ------ 3105 ValueError 3106 If `self.similarity` is None. 3107 3108 Notes 3109 ----- 3110 Builds a precomputed distance matrix combining correlation and euclidean distance, 3111 runs UMAP with metric='precomputed', then overlays cluster hulls (MeanShift + convex hull) 3112 and arrows to indicate nearest neighbors (minimal combined distance). 3113 """ 3114 3115 if self.similarity is None: 3116 raise ValueError( 3117 "Similarity data is missing. Please calculate similarity using self.estimating_similarity." 3118 ) 3119 3120 similarity_data = self.similarity 3121 3122 sim = similarity_data["correlation"] 3123 sim_scaled = (sim - sim.min()) / (sim.max() - sim.min()) 3124 eu_dist = similarity_data["euclidean_dist"] 3125 eu_dist_scaled = (eu_dist - eu_dist.min()) / (eu_dist.max() - eu_dist.min()) 3126 3127 similarity_data["combo_dist"] = (1 - sim_scaled) * eu_dist_scaled 3128 3129 # for nn target 3130 arrow_df = similarity_data.copy() 3131 arrow_df = similarity_data.loc[ 3132 similarity_data.groupby("cell1")["combo_dist"].idxmin() 3133 ] 3134 3135 cells = sorted(set(similarity_data["cell1"]) | set(similarity_data["cell2"])) 3136 combo_matrix = pd.DataFrame(0, index=cells, columns=cells, dtype=float) 3137 3138 for _, row in similarity_data.iterrows(): 3139 combo_matrix.loc[row["cell1"], row["cell2"]] = row["combo_dist"] 3140 combo_matrix.loc[row["cell2"], row["cell1"]] = row["combo_dist"] 3141 3142 umap_model = umap.UMAP( 3143 n_components=2, 3144 metric="precomputed", 3145 n_neighbors=n_neighbors, 3146 min_dist=min_dist, 3147 spread=spread, 3148 set_op_mix_ratio=set_op_mix_ratio, 3149 local_connectivity=local_connectivity, 3150 repulsion_strength=repulsion_strength, 3151 negative_sample_rate=negative_sample_rate, 3152 transform_seed=42, 3153 init="spectral", 3154 random_state=42, 3155 verbose=True, 3156 ) 3157 3158 coords = umap_model.fit_transform(combo_matrix.values) 3159 cell_names = list(combo_matrix.index) 3160 num_cells = len(cell_names) 3161 palette = sns.color_palette("tab20c", num_cells) 3162 3163 if "#" in cell_names[0]: 3164 avsets = set( 3165 [re.sub(".*# ", "", x) for x in similarity_data["cell1"]] 3166 + [re.sub(".*# ", "", x) for x in similarity_data["cell2"]] 3167 ) 3168 num_sets = len(avsets) 3169 color_indices = [i * len(palette) // num_sets for i in range(num_sets)] 3170 color_mapping_sets = { 3171 set_name: palette[i] for i, set_name in zip(color_indices, avsets) 3172 } 3173 color_mapping = { 3174 name: color_mapping_sets[re.sub(".*# ", "", name)] 3175 for i, name in enumerate(cell_names) 3176 } 3177 else: 3178 color_mapping = {name: palette[i] for i, name in enumerate(cell_names)} 3179 3180 meanshift = MeanShift(bandwidth=bandwidth) 3181 labels = meanshift.fit_predict(coords) 3182 3183 fig = plt.figure(figsize=(width, height)) 3184 ax = plt.gca() 3185 3186 unique_labels = set(labels) 3187 cluster_palette = sns.color_palette("hls", len(unique_labels)) 3188 3189 for label in unique_labels: 3190 if label == -1: 3191 continue 3192 cluster_coords = coords[labels == label] 3193 if len(cluster_coords) < 3: 3194 continue 3195 3196 hull = ConvexHull(cluster_coords) 3197 hull_points = cluster_coords[hull.vertices] 3198 3199 centroid = np.mean(hull_points, axis=0) 3200 expanded = hull_points + 0.05 * (hull_points - centroid) 3201 3202 poly = Polygon( 3203 expanded, 3204 closed=True, 3205 facecolor=cluster_palette[label], 3206 edgecolor="none", 3207 alpha=0.2, 3208 zorder=1, 3209 ) 3210 ax.add_patch(poly) 3211 3212 texts = [] 3213 for i, (x, y) in enumerate(coords): 3214 plt.scatter( 3215 x, 3216 y, 3217 s=point_size, 3218 color=color_mapping[cell_names[i]], 3219 edgecolors="black", 3220 linewidths=0.5, 3221 zorder=2, 3222 ) 3223 texts.append( 3224 ax.text( 3225 x, y, str(i), ha="center", va="center", fontsize=8, color="black" 3226 ) 3227 ) 3228 3229 def dist(p1, p2): 3230 return np.sqrt((p1[0] - p2[0]) ** 2 + (p1[1] - p2[1]) ** 2) 3231 3232 texts_to_adjust = [] 3233 for i, t1 in enumerate(texts): 3234 for j, t2 in enumerate(texts): 3235 if i >= j: 3236 continue 3237 d = dist( 3238 (t1.get_position()[0], t1.get_position()[1]), 3239 (t2.get_position()[0], t2.get_position()[1]), 3240 ) 3241 if d < threshold: 3242 if t1 not in texts_to_adjust: 3243 texts_to_adjust.append(t1) 3244 if t2 not in texts_to_adjust: 3245 texts_to_adjust.append(t2) 3246 3247 adjust_text( 3248 texts_to_adjust, 3249 expand_text=(1.0, 1.0), 3250 force_text=0.9, 3251 arrowprops=dict(arrowstyle="-", color="gray", lw=0.1), 3252 ax=ax, 3253 ) 3254 3255 for _, row in arrow_df.iterrows(): 3256 try: 3257 idx1 = cell_names.index(row["cell1"]) 3258 idx2 = cell_names.index(row["cell2"]) 3259 except ValueError: 3260 continue 3261 x1, y1 = coords[idx1] 3262 x2, y2 = coords[idx2] 3263 arrow = FancyArrowPatch( 3264 (x1, y1), 3265 (x2, y2), 3266 arrowstyle="->", 3267 color="gray", 3268 linewidth=1.5, 3269 alpha=0.5, 3270 mutation_scale=12, 3271 zorder=0, 3272 ) 3273 ax.add_patch(arrow) 3274 3275 if set_info is False and " # " in cell_names[0]: 3276 3277 legend_elements = [ 3278 Patch( 3279 facecolor=color_mapping[name], 3280 edgecolor="black", 3281 label=f"{i} – {re.sub(' #.*', '', name)}", 3282 ) 3283 for i, name in enumerate(cell_names) 3284 ] 3285 3286 else: 3287 3288 legend_elements = [ 3289 Patch( 3290 facecolor=color_mapping[name], 3291 edgecolor="black", 3292 label=f"{i} – {name}", 3293 ) 3294 for i, name in enumerate(cell_names) 3295 ] 3296 3297 plt.legend( 3298 handles=legend_elements, 3299 title="Cells", 3300 bbox_to_anchor=(1.05, 1), 3301 loc="upper left", 3302 ncol=legend_split, 3303 ) 3304 3305 plt.xlabel("UMAP 1") 3306 plt.ylabel("UMAP 2") 3307 plt.grid(False) 3308 plt.show() 3309 3310 return fig
Create a spatial UMAP-like visualization of similarity relationships between samples.
Parameters
set_info : bool, default True If True, retain set information in labels.
bandwidth : float, default 1 Bandwidth used by MeanShift for clustering polygons.
point_size : float, default 100 Size of scatter points.
legend_split : int, default 2 Number of columns in legend.
n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate : parameters passed to UMAP.
threshold : float, default 0.1 Minimum text distance for label adjustment to avoid overlap.
width : int, default 12 Figure width.
height : int, default 10 Figure height.
Returns
matplotlib.figure.Figure
Raises
ValueError
If self.similarity is None.
Notes
Builds a precomputed distance matrix combining correlation and euclidean distance, runs UMAP with metric='precomputed', then overlays cluster hulls (MeanShift + convex hull) and arrows to indicate nearest neighbors (minimal combined distance).
def
subcluster_prepare(self, features: list, cluster: str):
3314 def subcluster_prepare(self, features: list, cluster: str): 3315 """ 3316 Prepare a `Clustering` object for subcluster analysis on a selected parent cluster. 3317 3318 Parameters 3319 ---------- 3320 features : list 3321 Features to include for subcluster analysis. 3322 3323 cluster : str 3324 Parent cluster name (used to select matching cells). 3325 3326 Update 3327 ------ 3328 Initializes `self.subclusters_` as a new `Clustering` instance containing the 3329 reduced data for the given cluster and stores `current_features` and `current_cluster`. 3330 """ 3331 3332 dat = self.normalized_data 3333 dat.columns = list(self.input_metadata["cell_names"]) 3334 3335 dat = reduce_data(self.normalized_data, features=features, names=[cluster]) 3336 3337 self.subclusters_ = Clustering(data=dat, metadata=None) 3338 3339 self.subclusters_.current_features = features 3340 self.subclusters_.current_cluster = cluster
Prepare a Clustering object for subcluster analysis on a selected parent cluster.
Parameters
features : list Features to include for subcluster analysis.
cluster : str Parent cluster name (used to select matching cells).
Update
Initializes self.subclusters_ as a new Clustering instance containing the
reduced data for the given cluster and stores current_features and current_cluster.
def
define_subclusters( self, umap_num: int = 2, eps: float = 0.5, min_samples: int = 10, n_neighbors: int = 5, min_dist: float = 0.1, spread: float = 1.0, set_op_mix_ratio: float = 1.0, local_connectivity: int = 1, repulsion_strength: float = 1.0, negative_sample_rate: int = 5, width=8, height=6):
3342 def define_subclusters( 3343 self, 3344 umap_num: int = 2, 3345 eps: float = 0.5, 3346 min_samples: int = 10, 3347 n_neighbors: int = 5, 3348 min_dist: float = 0.1, 3349 spread: float = 1.0, 3350 set_op_mix_ratio: float = 1.0, 3351 local_connectivity: int = 1, 3352 repulsion_strength: float = 1.0, 3353 negative_sample_rate: int = 5, 3354 width=8, 3355 height=6, 3356 ): 3357 """ 3358 Compute UMAP and DBSCAN clustering within a previously prepared subcluster dataset. 3359 3360 Parameters 3361 ---------- 3362 umap_num : int, default 2 3363 Number of UMAP dimensions to compute. 3364 3365 eps : float, default 0.5 3366 DBSCAN eps parameter. 3367 3368 min_samples : int, default 10 3369 DBSCAN min_samples parameter. 3370 3371 n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate, width, height : 3372 Additional parameters passed to UMAP / plotting / MeanShift as appropriate. 3373 3374 Update 3375 ------- 3376 Stores cluster labels in `self.subclusters_.subclusters`. 3377 3378 Raises 3379 ------ 3380 RuntimeError 3381 If `self.subclusters_` has not been prepared. 3382 """ 3383 3384 if self.subclusters_ is None: 3385 raise RuntimeError( 3386 "Nothing to return. 'self.subcluster_prepare' was not conducted!" 3387 ) 3388 3389 self.subclusters_.perform_UMAP( 3390 factorize=False, 3391 umap_num=umap_num, 3392 pc_num=0, 3393 harmonized=False, 3394 n_neighbors=n_neighbors, 3395 min_dist=min_dist, 3396 spread=spread, 3397 set_op_mix_ratio=set_op_mix_ratio, 3398 local_connectivity=local_connectivity, 3399 repulsion_strength=repulsion_strength, 3400 negative_sample_rate=negative_sample_rate, 3401 width=width, 3402 height=height, 3403 ) 3404 3405 fig = self.subclusters_.find_clusters_UMAP( 3406 umap_n=umap_num, 3407 eps=eps, 3408 min_samples=min_samples, 3409 width=width, 3410 height=height, 3411 ) 3412 3413 clusters = self.subclusters_.return_clusters(clusters="umap") 3414 3415 self.subclusters_.subclusters = [str(x) for x in list(clusters)] 3416 3417 return fig
Compute UMAP and DBSCAN clustering within a previously prepared subcluster dataset.
Parameters
umap_num : int, default 2 Number of UMAP dimensions to compute.
eps : float, default 0.5 DBSCAN eps parameter.
min_samples : int, default 10 DBSCAN min_samples parameter.
n_neighbors, min_dist, spread, set_op_mix_ratio, local_connectivity, repulsion_strength, negative_sample_rate, width, height : Additional parameters passed to UMAP / plotting / MeanShift as appropriate.
Update
Stores cluster labels in self.subclusters_.subclusters.
Raises
RuntimeError
If self.subclusters_ has not been prepared.
def
subcluster_features_dot( self, colors='viridis', hclust='complete', scale=False, scale_axis: Literal['x', 'y'] = 'y', img_width=3, img_high=5, label_size=6, size_scale=70, y_lab='Genes', legend_lab='normalized', bbox_to_anchor_scale: int = 25, bbox_to_anchor_perc: tuple = (0.91, 0.63)):
3419 def subcluster_features_dot( 3420 self, 3421 colors="viridis", 3422 hclust="complete", 3423 scale=False, 3424 scale_axis: Literal["x", "y"] = "y", 3425 img_width=3, 3426 img_high=5, 3427 label_size=6, 3428 size_scale=70, 3429 y_lab="Genes", 3430 legend_lab="normalized", 3431 bbox_to_anchor_scale: int = 25, 3432 bbox_to_anchor_perc: tuple = (0.91, 0.63), 3433 ): 3434 """ 3435 Create a features-dot visualization for the subclusters (averaged and occurrence). 3436 3437 Parameters 3438 ---------- 3439 colors : str, default 'viridis' 3440 Colormap name passed to `features_dot`. 3441 3442 hclust : str or None 3443 Hierarchical clustering linkage to order rows/columns. 3444 3445 scale: bool, default False 3446 If True, expression data will be scaled (0–1) across the rows (features). 3447 3448 scale_axis : {'x', 'y'}, default 'y' 3449 The axis along which to scale the expression data to the [0, 1] range: 3450 - 'y': Scales across columns (per sample / cell). 3451 - 'x': Scales across rows (per feature / gene). 3452 Only effective if `scale=True`. 3453 3454 img_width, img_high : float 3455 Figure size. 3456 3457 label_size : int 3458 Font size for labels. 3459 3460 size_scale : int 3461 Bubble size scaling. 3462 3463 y_lab : str 3464 X axis label. 3465 3466 legend_lab : str 3467 Colorbar label. 3468 3469 bbox_to_anchor_scale : int, default=25 3470 Vertical scale (percentage) for positioning the colorbar. 3471 3472 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3473 Anchor position for the size legend (percent bubble legend). 3474 3475 Returns 3476 ------- 3477 matplotlib.figure.Figure 3478 3479 Raises 3480 ------ 3481 RuntimeError 3482 If subcluster preparation/definition has not been run. 3483 """ 3484 3485 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3486 raise RuntimeError( 3487 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3488 ) 3489 3490 dat = self.normalized_data 3491 dat.columns = list(self.input_metadata["cell_names"]) 3492 3493 dat = reduce_data( 3494 self.normalized_data, 3495 features=self.subclusters_.current_features, 3496 names=[self.subclusters_.current_cluster], 3497 ) 3498 3499 dat.columns = self.subclusters_.subclusters 3500 3501 avg = average(dat) 3502 occ = occurrence(dat) 3503 3504 scatter = features_dot( 3505 expression_data=avg, 3506 occurence_data=occ, 3507 features=None, 3508 scale=scale, 3509 scale_axis=scale_axis, 3510 metadata_list=None, 3511 colors=colors, 3512 hclust=hclust, 3513 img_width=img_width, 3514 img_high=img_high, 3515 label_size=label_size, 3516 size_scale=size_scale, 3517 y_lab=y_lab, 3518 legend_lab=legend_lab, 3519 bbox_to_anchor_scale=bbox_to_anchor_scale, 3520 bbox_to_anchor_perc=bbox_to_anchor_perc, 3521 ) 3522 3523 return scatter
Create a features-dot visualization for the subclusters (averaged and occurrence).
Parameters
colors : str, default 'viridis'
Colormap name passed to features_dot.
hclust : str or None Hierarchical clustering linkage to order rows/columns.
scale: bool, default False If True, expression data will be scaled (0–1) across the rows (features).
scale_axis : {'x', 'y'}, default 'y'
The axis along which to scale the expression data to the [0, 1] range:
- 'y': Scales across columns (per sample / cell).
- 'x': Scales across rows (per feature / gene).
Only effective if scale=True.
img_width, img_high : float Figure size.
label_size : int Font size for labels.
size_scale : int Bubble size scaling.
y_lab : str X axis label.
legend_lab : str Colorbar label.
bbox_to_anchor_scale : int, default=25 Vertical scale (percentage) for positioning the colorbar.
bbox_to_anchor_perc : tuple, default=(0.91, 0.63) Anchor position for the size legend (percent bubble legend).
Returns
matplotlib.figure.Figure
Raises
RuntimeError If subcluster preparation/definition has not been run.
def
subcluster_DEG_dot( self, top_n=3, min_exp=0, min_pct=0.25, p_val=0.05, colors='viridis', hclust='complete', scale=False, scale_axis: Literal['x', 'y'] = 'y', img_width=3, img_high=5, label_size=6, size_scale=70, y_lab='Genes', legend_lab='normalized', bbox_to_anchor_scale: int = 25, bbox_to_anchor_perc: tuple = (0.91, 0.63), n_proc=10):
3525 def subcluster_DEG_dot( 3526 self, 3527 top_n=3, 3528 min_exp=0, 3529 min_pct=0.25, 3530 p_val=0.05, 3531 colors="viridis", 3532 hclust="complete", 3533 scale=False, 3534 scale_axis: Literal["x", "y"] = "y", 3535 img_width=3, 3536 img_high=5, 3537 label_size=6, 3538 size_scale=70, 3539 y_lab="Genes", 3540 legend_lab="normalized", 3541 bbox_to_anchor_scale: int = 25, 3542 bbox_to_anchor_perc: tuple = (0.91, 0.63), 3543 n_proc=10, 3544 ): 3545 """ 3546 Plot top differential features (DEGs) for subclusters as a features-dot. 3547 3548 Parameters 3549 ---------- 3550 top_n : int, default 3 3551 Number of top features per subcluster to show. 3552 3553 min_exp : float, default 0 3554 Minimum expression threshold passed to `statistic`. 3555 3556 min_pct : float, default 0.25 3557 Minimum percent expressed in target group. 3558 3559 p_val: float, default 0.05 3560 Maximum p-value for visualizing features. 3561 3562 n_proc : int, default 10 3563 Parallel jobs used for DEG calculation. 3564 3565 scale: bool, default False 3566 If True, expression_data will be scaled (0–1) across the rows (features). 3567 3568 scale_axis : {'x', 'y'}, default 'y' 3569 The axis along which to scale the expression data to the [0, 1] range: 3570 - 'y': Scales across columns (per sample / cell). 3571 - 'x': Scales across rows (per feature / gene). 3572 Only effective if `scale=True`. 3573 3574 colors : str, default='viridis' 3575 Colormap for expression values. 3576 3577 hclust : str or None, default='complete' 3578 Linkage method for hierarchical clustering. If None, no clustering 3579 is performed. 3580 3581 img_width : int or float, default=8 3582 Width of the plot in inches. 3583 3584 img_high : int or float, default=5 3585 Height of the plot in inches. 3586 3587 label_size : int, default=10 3588 Font size for axis labels and ticks. 3589 3590 size_scale : int or float, default=100 3591 Scaling factor for bubble sizes. 3592 3593 y_lab : str, default='Genes' 3594 Label for the x-axis. 3595 3596 legend_lab : str, default='normalized' 3597 Label for the colorbar legend. 3598 3599 bbox_to_anchor_scale : int, default=25 3600 Vertical scale (percentage) for positioning the colorbar. 3601 3602 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3603 Anchor position for the size legend (percent bubble legend). 3604 3605 Returns 3606 ------- 3607 matplotlib.figure.Figure 3608 3609 Raises 3610 ------ 3611 RuntimeError 3612 If subcluster preparation/definition has not been run. 3613 3614 Notes 3615 ----- 3616 Internally calls `calc_DEG` (or equivalent) to obtain statistics, filters 3617 by p-value and effect-size, selects top features per valid group and plots them. 3618 """ 3619 3620 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3621 raise RuntimeError( 3622 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3623 ) 3624 3625 dat = self.normalized_data 3626 dat.columns = list(self.input_metadata["cell_names"]) 3627 3628 dat = reduce_data( 3629 self.normalized_data, names=[self.subclusters_.current_cluster] 3630 ) 3631 3632 dat.columns = self.subclusters_.subclusters 3633 3634 deg_stats = calc_DEG( 3635 dat, 3636 metadata_list=None, 3637 entities="All", 3638 sets=None, 3639 min_exp=min_exp, 3640 min_pct=min_pct, 3641 n_proc=n_proc, 3642 ) 3643 3644 deg_stats = deg_stats[deg_stats["p_val"] <= p_val] 3645 deg_stats = deg_stats[deg_stats["log(FC)"] > 0] 3646 3647 deg_stats = ( 3648 deg_stats.sort_values( 3649 ["valid_group", "esm", "log(FC)"], ascending=[True, False, False] 3650 ) 3651 .groupby("valid_group") 3652 .head(top_n) 3653 ) 3654 3655 dat = reduce_data(dat, features=list(set(deg_stats["feature"]))) 3656 3657 avg = average(dat) 3658 occ = occurrence(dat) 3659 3660 scatter = features_dot( 3661 expression_data=avg, 3662 occurence_data=occ, 3663 features=None, 3664 metadata_list=None, 3665 scale_axis=scale_axis, 3666 scale=scale, 3667 colors=colors, 3668 hclust=hclust, 3669 img_width=img_width, 3670 img_high=img_high, 3671 label_size=label_size, 3672 size_scale=size_scale, 3673 y_lab=y_lab, 3674 legend_lab=legend_lab, 3675 bbox_to_anchor_scale=bbox_to_anchor_scale, 3676 bbox_to_anchor_perc=bbox_to_anchor_perc, 3677 ) 3678 3679 return scatter
Plot top differential features (DEGs) for subclusters as a features-dot.
Parameters
top_n : int, default 3 Number of top features per subcluster to show.
min_exp : float, default 0
Minimum expression threshold passed to statistic.
min_pct : float, default 0.25 Minimum percent expressed in target group.
p_val: float, default 0.05 Maximum p-value for visualizing features.
n_proc : int, default 10 Parallel jobs used for DEG calculation.
scale: bool, default False If True, expression_data will be scaled (0–1) across the rows (features).
scale_axis : {'x', 'y'}, default 'y'
The axis along which to scale the expression data to the [0, 1] range:
- 'y': Scales across columns (per sample / cell).
- 'x': Scales across rows (per feature / gene).
Only effective if scale=True.
colors : str, default='viridis' Colormap for expression values.
hclust : str or None, default='complete' Linkage method for hierarchical clustering. If None, no clustering is performed.
img_width : int or float, default=8 Width of the plot in inches.
img_high : int or float, default=5 Height of the plot in inches.
label_size : int, default=10 Font size for axis labels and ticks.
size_scale : int or float, default=100 Scaling factor for bubble sizes.
y_lab : str, default='Genes' Label for the x-axis.
legend_lab : str, default='normalized' Label for the colorbar legend.
bbox_to_anchor_scale : int, default=25 Vertical scale (percentage) for positioning the colorbar.
bbox_to_anchor_perc : tuple, default=(0.91, 0.63) Anchor position for the size legend (percent bubble legend).
Returns
matplotlib.figure.Figure
Raises
RuntimeError If subcluster preparation/definition has not been run.
Notes
Internally calls calc_DEG (or equivalent) to obtain statistics, filters
by p-value and effect-size, selects top features per valid group and plots them.
def
accept_subclusters(self):
3681 def accept_subclusters(self): 3682 """ 3683 Commit subcluster labels into the main `input_metadata` by renaming cell names. 3684 3685 The method replaces occurrences of the parent cluster name in `self.input_metadata['cell_names']` 3686 with the expanded names that include subcluster suffixes (via `add_subnames`), 3687 then clears `self.subclusters_`. 3688 3689 Update 3690 ------ 3691 Modifies `self.input_metadata['cell_names']`. 3692 3693 Resets `self.subclusters_` to None. 3694 3695 Raises 3696 ------ 3697 RuntimeError 3698 If `self.subclusters_` is not defined or subclusters were not computed. 3699 """ 3700 3701 if self.subclusters_ is None or self.subclusters_.subclusters is None: 3702 raise RuntimeError( 3703 "Nothing to return. 'self.subcluster_prepare' -> 'self.define_subclusters' pip was not conducted!" 3704 ) 3705 3706 new_meta = add_subnames( 3707 list(self.input_metadata["cell_names"]), 3708 parent_name=self.subclusters_.current_cluster, 3709 new_clusters=self.subclusters_.subclusters, 3710 ) 3711 3712 self.input_metadata["cell_names"] = new_meta 3713 3714 self.subclusters_ = None
Commit subcluster labels into the main input_metadata by renaming cell names.
The method replaces occurrences of the parent cluster name in self.input_metadata['cell_names']
with the expanded names that include subcluster suffixes (via add_subnames),
then clears self.subclusters_.
Update
Modifies self.input_metadata['cell_names'].
Resets self.subclusters_ to None.
Raises
RuntimeError
If self.subclusters_ is not defined or subclusters were not computed.
def
dot_plot( self, names: list | None = None, features: list | None = None, name_slot: str = 'cell_names', scale=True, scale_axis: Literal['x', 'y'] = 'y', colors='viridis', hclust=None, img_width=15, img_high=1, label_size=10, size_scale=200, y_lab='Genes', legend_lab='log(CPM + 1)', set_box_size: float | int = 5, set_box_high: float | int = 5, bbox_to_anchor_scale=25, bbox_to_anchor_perc=(0.9, -0.24), bbox_to_anchor_group=(1.01, 0.4)):
3716 def dot_plot( 3717 self, 3718 names: list | None = None, 3719 features: list | None = None, 3720 name_slot: str = "cell_names", 3721 scale=True, 3722 scale_axis: Literal["x", "y"] = "y", 3723 colors="viridis", 3724 hclust=None, 3725 img_width=15, 3726 img_high=1, 3727 label_size=10, 3728 size_scale=200, 3729 y_lab="Genes", 3730 legend_lab="log(CPM + 1)", 3731 set_box_size: float | int = 5, 3732 set_box_high: float | int = 5, 3733 bbox_to_anchor_scale=25, 3734 bbox_to_anchor_perc=(0.90, -0.24), 3735 bbox_to_anchor_group=(1.01, 0.4), 3736 ): 3737 """ 3738 Create a dot plot of selected features across samples inside project. 3739 3740 Each point represents a feature-sample pair, where the color encodes the 3741 expression value and the size encodes occurrence or relative abundance. 3742 Optionally, hierarchical clustering can be applied to order rows and columns. 3743 3744 Parameters 3745 ---------- 3746 names : list, str, or None 3747 Names of samples to include. If None, all samples are considered. 3748 3749 features : list, str, or None 3750 Names of features to include. If None, all features are considered. 3751 3752 name_slot : str 3753 Column in metadata to use as sample names. 3754 3755 scale: bool, default False 3756 If True, expression_data will be scaled (0–1) across the rows (features). 3757 3758 scale_axis : {'x', 'y'}, default 'y' 3759 The axis along which to scale the expression data to the [0, 1] range: 3760 - 'y': Scales across columns (per sample / cell). 3761 - 'x': Scales across rows (per feature / gene). 3762 Only effective if `scale=True`. 3763 3764 colors : str, default='viridis' 3765 Colormap for expression values. 3766 3767 hclust : str or None, default='complete' 3768 Linkage method for hierarchical clustering. If None, no clustering 3769 is performed. 3770 3771 img_width : int or float, default=8 3772 Width of the plot in inches. 3773 3774 img_high : int or float, default=5 3775 Height of the plot in inches. 3776 3777 label_size : int, default=10 3778 Font size for axis labels and ticks. 3779 3780 size_scale : int or float, default=100 3781 Scaling factor for bubble sizes. 3782 3783 y_lab : str, default='Genes' 3784 Label for the x-axis. 3785 3786 legend_lab : str, default='log(CPM + 1)' 3787 Label for the colorbar legend. 3788 3789 bbox_to_anchor_scale : int, default=25 3790 Vertical scale (percentage) for positioning the colorbar. 3791 3792 bbox_to_anchor_perc : tuple, default=(0.91, 0.63) 3793 Anchor position for the size legend (percent bubble legend). 3794 3795 bbox_to_anchor_group : tuple, default=(1.01, 0.4) 3796 Anchor position for the group legend. 3797 3798 Returns 3799 ------- 3800 matplotlib.figure.Figure 3801 The generated dot plot figure. 3802 3803 Notes 3804 ----- 3805 Colors represent expression values normalized to the colormap. 3806 """ 3807 3808 prtd, met = self.get_partial_data( 3809 names=names, features=features, name_slot=name_slot, inc_metadata=True 3810 ) 3811 3812 prtd.columns = prtd.columns + "#" + met["sets"] 3813 3814 prtd_avg = average(prtd) 3815 3816 meta_sets = [re.sub(".*#", "", x) for x in prtd_avg.columns] 3817 3818 prtd_avg.columns = [re.sub("#.*", "", x) for x in prtd_avg.columns] 3819 3820 prtd_occ = occurrence(prtd) 3821 3822 prtd_occ.columns = [re.sub("#.*", "", x) for x in prtd_occ.columns] 3823 3824 fig_scatter = features_dot( 3825 expression_data=prtd_avg, 3826 occurence_data=prtd_occ, 3827 scale=scale, 3828 scale_axis=scale_axis, 3829 features=None, 3830 metadata_list=meta_sets, 3831 colors=colors, 3832 hclust=hclust, 3833 img_width=img_width, 3834 img_high=img_high, 3835 label_size=label_size, 3836 size_scale=size_scale, 3837 y_lab=y_lab, 3838 legend_lab=legend_lab, 3839 set_box_size=set_box_size, 3840 set_box_high=set_box_high, 3841 bbox_to_anchor_scale=bbox_to_anchor_scale, 3842 bbox_to_anchor_perc=bbox_to_anchor_perc, 3843 bbox_to_anchor_group=bbox_to_anchor_group, 3844 ) 3845 3846 return fig_scatter
Create a dot plot of selected features across samples inside project.
Each point represents a feature-sample pair, where the color encodes the expression value and the size encodes occurrence or relative abundance. Optionally, hierarchical clustering can be applied to order rows and columns.
Parameters
names : list, str, or None Names of samples to include. If None, all samples are considered.
features : list, str, or None Names of features to include. If None, all features are considered.
name_slot : str Column in metadata to use as sample names.
scale: bool, default False If True, expression_data will be scaled (0–1) across the rows (features).
scale_axis : {'x', 'y'}, default 'y'
The axis along which to scale the expression data to the [0, 1] range:
- 'y': Scales across columns (per sample / cell).
- 'x': Scales across rows (per feature / gene).
Only effective if scale=True.
colors : str, default='viridis' Colormap for expression values.
hclust : str or None, default='complete' Linkage method for hierarchical clustering. If None, no clustering is performed.
img_width : int or float, default=8 Width of the plot in inches.
img_high : int or float, default=5 Height of the plot in inches.
label_size : int, default=10 Font size for axis labels and ticks.
size_scale : int or float, default=100 Scaling factor for bubble sizes.
y_lab : str, default='Genes' Label for the x-axis.
legend_lab : str, default='log(CPM + 1)' Label for the colorbar legend.
bbox_to_anchor_scale : int, default=25 Vertical scale (percentage) for positioning the colorbar.
bbox_to_anchor_perc : tuple, default=(0.91, 0.63) Anchor position for the size legend (percent bubble legend).
bbox_to_anchor_group : tuple, default=(1.01, 0.4) Anchor position for the group legend.
Returns
matplotlib.figure.Figure The generated dot plot figure.
Notes
Colors represent expression values normalized to the colormap.
def
data_composition( self, features_count: list | None, name_slot: str = 'cell_names', set_sep: bool = True):
3848 def data_composition( 3849 self, 3850 features_count: list | None, 3851 name_slot: str = "cell_names", 3852 set_sep: bool = True, 3853 ): 3854 """ 3855 Compute composition of cell types in data set. 3856 3857 This function counts the occurrences of specific cells (e.g., cell types, subtypes) 3858 within metadata entries, calculates their relative percentages, and stores 3859 the results in `self.composition_data`. 3860 3861 Parameters 3862 ---------- 3863 features_count : list or None 3864 List of features (part or full names) to be counted. 3865 If None, all unique elements from the specified `name_slot` metadata field are used. 3866 3867 name_slot : str, default 'cell_names' 3868 Metadata field containing sample identifiers or labels. 3869 3870 set_sep : bool, default True 3871 If True and multiple sets are present in metadata, compute composition 3872 separately for each set. 3873 3874 Update 3875 ------- 3876 Stores results in `self.composition_data` as a pandas DataFrame with: 3877 - 'name': feature name 3878 - 'n': number of occurrences 3879 - 'pct': percentage of occurrences 3880 - 'set' (if applicable): dataset identifier 3881 """ 3882 3883 validated_list = list(self.input_metadata[name_slot]) 3884 sets = list(self.input_metadata["sets"]) 3885 3886 if features_count is None: 3887 features_count = list(set(self.input_metadata[name_slot])) 3888 3889 if set_sep and len(set(sets)) > 1: 3890 3891 final_res = pd.DataFrame() 3892 3893 for s in set(sets): 3894 print(s) 3895 3896 mask = [True if s == x else False for x in sets] 3897 3898 tmp_val_list = np.array(validated_list) 3899 3900 tmp_val_list = list(tmp_val_list[mask]) 3901 3902 res_dict = {"name": [], "n": [], "set": []} 3903 3904 for f in tqdm(features_count): 3905 res_dict["n"].append( 3906 sum(1 for element in tmp_val_list if f in element) 3907 ) 3908 res_dict["name"].append(f) 3909 res_dict["set"].append(s) 3910 res = pd.DataFrame(res_dict) 3911 res["pct"] = res["n"] / sum(res["n"]) * 100 3912 res["pct"] = res["pct"].round(2) 3913 3914 final_res = pd.concat([final_res, res]) 3915 3916 res = final_res.sort_values(["set", "pct"], ascending=[True, False]) 3917 3918 else: 3919 3920 res_dict = {"name": [], "n": []} 3921 3922 for f in tqdm(features_count): 3923 res_dict["n"].append( 3924 sum(1 for element in validated_list if f in element) 3925 ) 3926 res_dict["name"].append(f) 3927 3928 res = pd.DataFrame(res_dict) 3929 res["pct"] = res["n"] / sum(res["n"]) * 100 3930 res["pct"] = res["pct"].round(2) 3931 3932 res = res.sort_values("pct", ascending=False) 3933 3934 self.composition_data = res
Compute composition of cell types in data set.
This function counts the occurrences of specific cells (e.g., cell types, subtypes)
within metadata entries, calculates their relative percentages, and stores
the results in self.composition_data.
Parameters
features_count : list or None
List of features (part or full names) to be counted.
If None, all unique elements from the specified name_slot metadata field are used.
name_slot : str, default 'cell_names' Metadata field containing sample identifiers or labels.
set_sep : bool, default True If True and multiple sets are present in metadata, compute composition separately for each set.
Update
Stores results in self.composition_data as a pandas DataFrame with:
- 'name': feature name
- 'n': number of occurrences
- 'pct': percentage of occurrences
- 'set' (if applicable): dataset identifier
def
composition_pie( self, width=6, height=6, font_size=15, cmap: str = 'tab20', legend_split_col: int = 1, offset_labels: float | int = 0.5, legend_bbox: tuple = (1.15, 0.95)):
3936 def composition_pie( 3937 self, 3938 width=6, 3939 height=6, 3940 font_size=15, 3941 cmap: str = "tab20", 3942 legend_split_col: int = 1, 3943 offset_labels: float | int = 0.5, 3944 legend_bbox: tuple = (1.15, 0.95), 3945 ): 3946 """ 3947 Visualize the composition of cell lineages using pie charts. 3948 3949 Generates pie charts showing the relative proportions of features stored 3950 in `self.composition_data`. If multiple sets are present, a separate 3951 chart is drawn for each set. 3952 3953 Parameters 3954 ---------- 3955 width : int, default 6 3956 Width of the figure. 3957 3958 height : int, default 6 3959 Height of the figure (applied per set if multiple sets are plotted). 3960 3961 font_size : int, default 15 3962 Font size for labels and annotations. 3963 3964 cmap : str, default 'tab20' 3965 Colormap used for pie slices. 3966 3967 legend_split_col : int, default 1 3968 Number of columns in the legend. 3969 3970 offset_labels : float or int, default 0.5 3971 Spacing offset for label placement relative to pie slices. 3972 3973 legend_bbox : tuple, default (1.15, 0.95) 3974 Bounding box anchor position for the legend. 3975 3976 Returns 3977 ------- 3978 matplotlib.figure.Figure 3979 Pie chart visualization of composition data. 3980 """ 3981 3982 df = self.composition_data 3983 3984 if "set" in df.columns and len(set(df["set"])) > 1: 3985 3986 sets = list(set(df["set"])) 3987 fig, axes = plt.subplots(len(sets), 1, figsize=(width, height * len(sets))) 3988 3989 all_wedges = [] 3990 cmap = plt.get_cmap(cmap) 3991 3992 set_nam = len(set(df["name"])) 3993 3994 legend_labels = list(set(df["name"])) 3995 3996 colors = [cmap(i / set_nam) for i in range(set_nam)] 3997 3998 cmap_dict = dict(zip(legend_labels, colors)) 3999 4000 for idx, s in enumerate(sets): 4001 ax = axes[idx] 4002 tmp_df = df[df["set"] == s].reset_index(drop=True) 4003 4004 labels = [f"{row['pct']:.1f}%" for _, row in tmp_df.iterrows()] 4005 4006 wedges, _ = ax.pie( 4007 tmp_df["n"], 4008 startangle=90, 4009 labeldistance=1.05, 4010 colors=[cmap_dict[x] for x in tmp_df["name"]], 4011 wedgeprops={"linewidth": 0.5, "edgecolor": "black"}, 4012 ) 4013 4014 all_wedges.extend(wedges) 4015 4016 kw = dict(arrowprops=dict(arrowstyle="-"), zorder=0, va="center") 4017 n = 0 4018 for i, p in enumerate(wedges): 4019 ang = (p.theta2 - p.theta1) / 2.0 + p.theta1 4020 y = np.sin(np.deg2rad(ang)) 4021 x = np.cos(np.deg2rad(ang)) 4022 horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] 4023 connectionstyle = f"angle,angleA=0,angleB={ang}" 4024 kw["arrowprops"].update({"connectionstyle": connectionstyle}) 4025 if len(labels[i]) > 0: 4026 n += offset_labels 4027 ax.annotate( 4028 labels[i], 4029 xy=(x, y), 4030 xytext=(1.01 * x + (n * x / 4), 1.01 * y + (n * y / 4)), 4031 horizontalalignment=horizontalalignment, 4032 fontsize=font_size, 4033 weight="bold", 4034 **kw, 4035 ) 4036 4037 circle2 = plt.Circle((0, 0), 0.6, color="white", ec="black") 4038 ax.add_artist(circle2) 4039 4040 ax.text( 4041 0, 4042 0, 4043 f"{s}", 4044 ha="center", 4045 va="center", 4046 fontsize=font_size, 4047 weight="bold", 4048 ) 4049 4050 legend_handles = [ 4051 Patch(facecolor=cmap_dict[label], edgecolor="black", label=label) 4052 for label in legend_labels 4053 ] 4054 4055 fig.legend( 4056 handles=legend_handles, 4057 loc="center right", 4058 bbox_to_anchor=legend_bbox, 4059 ncol=legend_split_col, 4060 title="", 4061 ) 4062 4063 plt.tight_layout() 4064 plt.show() 4065 4066 else: 4067 4068 labels = [f"{row['pct']:.1f}%" for _, row in df.iterrows()] 4069 4070 legend_labels = [f"{row['name']}" for _, row in df.iterrows()] 4071 4072 cmap = plt.get_cmap(cmap) 4073 colors = [cmap(i / len(df)) for i in range(len(df))] 4074 4075 fig, ax = plt.subplots( 4076 figsize=(width, height), subplot_kw=dict(aspect="equal") 4077 ) 4078 4079 wedges, _ = ax.pie( 4080 df["n"], 4081 startangle=90, 4082 labeldistance=1.05, 4083 colors=colors, 4084 wedgeprops={"linewidth": 0.5, "edgecolor": "black"}, 4085 ) 4086 4087 kw = dict(arrowprops=dict(arrowstyle="-"), zorder=0, va="center") 4088 n = 0 4089 for i, p in enumerate(wedges): 4090 ang = (p.theta2 - p.theta1) / 2.0 + p.theta1 4091 y = np.sin(np.deg2rad(ang)) 4092 x = np.cos(np.deg2rad(ang)) 4093 horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))] 4094 connectionstyle = "angle,angleA=0,angleB={}".format(ang) 4095 kw["arrowprops"].update({"connectionstyle": connectionstyle}) 4096 if len(labels[i]) > 0: 4097 n += offset_labels 4098 4099 ax.annotate( 4100 labels[i], 4101 xy=(x, y), 4102 xytext=(1.01 * x + (n * x / 4), y * 1.01 + (n * y / 4)), 4103 horizontalalignment=horizontalalignment, 4104 fontsize=font_size, 4105 weight="bold", 4106 **kw, 4107 ) 4108 4109 circle2 = plt.Circle((0, 0), 0.6, color="white") 4110 circle2.set_edgecolor("black") 4111 4112 p = plt.gcf() 4113 p.gca().add_artist(circle2) 4114 4115 ax.legend( 4116 wedges, 4117 legend_labels, 4118 title="", 4119 loc="center left", 4120 bbox_to_anchor=legend_bbox, 4121 ncol=legend_split_col, 4122 ) 4123 4124 plt.show() 4125 4126 return fig
Visualize the composition of cell lineages using pie charts.
Generates pie charts showing the relative proportions of features stored
in self.composition_data. If multiple sets are present, a separate
chart is drawn for each set.
Parameters
width : int, default 6 Width of the figure.
height : int, default 6 Height of the figure (applied per set if multiple sets are plotted).
font_size : int, default 15 Font size for labels and annotations.
cmap : str, default 'tab20' Colormap used for pie slices.
legend_split_col : int, default 1 Number of columns in the legend.
offset_labels : float or int, default 0.5 Spacing offset for label placement relative to pie slices.
legend_bbox : tuple, default (1.15, 0.95) Bounding box anchor position for the legend.
Returns
matplotlib.figure.Figure Pie chart visualization of composition data.
def
bar_composition( self, cmap='tab20b', width=2, height=6, font_size=15, legend_split_col: int = 1, legend_bbox: tuple = (1.3, 1)):
4128 def bar_composition( 4129 self, 4130 cmap="tab20b", 4131 width=2, 4132 height=6, 4133 font_size=15, 4134 legend_split_col: int = 1, 4135 legend_bbox: tuple = (1.3, 1), 4136 ): 4137 """ 4138 Visualize the composition of cell lineages using bar plots. 4139 4140 Produces bar plots showing the distribution of features stored in 4141 `self.composition_data`. If multiple sets are present, a separate 4142 bar is drawn for each set. Percentages are annotated alongside the bars. 4143 4144 Parameters 4145 ---------- 4146 cmap : str, default 'tab20b' 4147 Colormap used for stacked bars. 4148 4149 width : int, default 2 4150 Width of each subplot (per set). 4151 4152 height : int, default 6 4153 Height of the figure. 4154 4155 font_size : int, default 15 4156 Font size for labels and annotations. 4157 4158 legend_split_col : int, default 1 4159 Number of columns in the legend. 4160 4161 legend_bbox : tuple, default (1.3, 1) 4162 Bounding box anchor position for the legend. 4163 4164 Returns 4165 ------- 4166 matplotlib.figure.Figure 4167 Stacked bar plot visualization of composition data. 4168 """ 4169 4170 df = self.composition_data 4171 df["num"] = range(1, len(df) + 1) 4172 4173 if "set" in df.columns and len(set(df["set"])) > 1: 4174 4175 sets = list(set(df["set"])) 4176 fig, axes = plt.subplots(1, len(sets), figsize=(width * len(sets), height)) 4177 4178 cmap = plt.get_cmap(cmap) 4179 4180 set_nam = len(set(df["name"])) 4181 4182 legend_labels = list(set(df["name"])) 4183 4184 colors = [cmap(i / set_nam) for i in range(set_nam)] 4185 4186 cmap_dict = dict(zip(legend_labels, colors)) 4187 4188 for idx, s in enumerate(sets): 4189 ax = axes[idx] 4190 4191 tmp_df = df[df["set"] == s].reset_index(drop=True) 4192 4193 values = tmp_df["n"].values 4194 total = sum(values) 4195 values = [v / total * 100 for v in values] 4196 values = [round(v, 2) for v in values] 4197 4198 idx_max = np.argmax(values) 4199 correction = 100 - sum(values) 4200 values[idx_max] += correction 4201 4202 names = tmp_df["name"].values 4203 perc = tmp_df["pct"].values 4204 nums = tmp_df["num"].values 4205 4206 bottom = 0 4207 centers = [] 4208 for name, num, val, color in zip(names, nums, values, colors): 4209 ax.bar(s, val, bottom=bottom, color=cmap_dict[name], label=name) 4210 centers.append(bottom + val / 2) 4211 bottom += val 4212 4213 y_positions = np.linspace(centers[0], centers[-1], len(centers)) 4214 x_text = -0.8 4215 4216 for y_label, y_center, pct, num in zip( 4217 y_positions, centers, perc, nums 4218 ): 4219 ax.annotate( 4220 f"{pct:.1f}%", 4221 xy=(0, y_center), 4222 xycoords="data", 4223 xytext=(x_text, y_label), 4224 textcoords="data", 4225 ha="right", 4226 va="center", 4227 fontsize=font_size, 4228 arrowprops=dict( 4229 arrowstyle="->", 4230 lw=1, 4231 color="black", 4232 connectionstyle="angle3,angleA=0,angleB=90", 4233 ), 4234 ) 4235 4236 ax.set_ylim(0, 100) 4237 ax.set_xlabel(s, fontsize=font_size) 4238 ax.xaxis.label.set_rotation(30) 4239 4240 ax.set_xticks([]) 4241 ax.set_yticks([]) 4242 for spine in ax.spines.values(): 4243 spine.set_visible(False) 4244 4245 legend_handles = [ 4246 Patch(facecolor=cmap_dict[label], edgecolor="black", label=label) 4247 for label in legend_labels 4248 ] 4249 4250 fig.legend( 4251 handles=legend_handles, 4252 loc="center right", 4253 bbox_to_anchor=legend_bbox, 4254 ncol=legend_split_col, 4255 title="", 4256 ) 4257 4258 plt.tight_layout() 4259 plt.show() 4260 4261 else: 4262 4263 cmap = plt.get_cmap(cmap) 4264 4265 colors = [cmap(i / len(df)) for i in range(len(df))] 4266 4267 fig, ax = plt.subplots(figsize=(width, height)) 4268 4269 values = df["n"].values 4270 names = df["name"].values 4271 perc = df["pct"].values 4272 nums = df["num"].values 4273 4274 bottom = 0 4275 centers = [] 4276 for name, num, val, color in zip(names, nums, values, colors): 4277 ax.bar(0, val, bottom=bottom, color=color, label=f"{num}) {name}") 4278 centers.append(bottom + val / 2) 4279 bottom += val 4280 4281 y_positions = np.linspace(centers[0], centers[-1], len(centers)) 4282 x_text = -0.8 4283 4284 for y_label, y_center, pct, num in zip(y_positions, centers, perc, nums): 4285 ax.annotate( 4286 f"{num}) {pct}", 4287 xy=(0, y_center), 4288 xycoords="data", 4289 xytext=(x_text, y_label), 4290 textcoords="data", 4291 ha="right", 4292 va="center", 4293 fontsize=9, 4294 arrowprops=dict( 4295 arrowstyle="->", 4296 lw=1, 4297 color="black", 4298 connectionstyle="angle3,angleA=0,angleB=90", 4299 ), 4300 ) 4301 4302 ax.set_xticks([]) 4303 ax.set_yticks([]) 4304 for spine in ax.spines.values(): 4305 spine.set_visible(False) 4306 4307 ax.legend( 4308 title="Legend", 4309 bbox_to_anchor=legend_bbox, 4310 loc="upper left", 4311 ncol=legend_split_col, 4312 ) 4313 4314 plt.tight_layout() 4315 plt.show() 4316 4317 return fig
Visualize the composition of cell lineages using bar plots.
Produces bar plots showing the distribution of features stored in
self.composition_data. If multiple sets are present, a separate
bar is drawn for each set. Percentages are annotated alongside the bars.
Parameters
cmap : str, default 'tab20b' Colormap used for stacked bars.
width : int, default 2 Width of each subplot (per set).
height : int, default 6 Height of the figure.
font_size : int, default 15 Font size for labels and annotations.
legend_split_col : int, default 1 Number of columns in the legend.
legend_bbox : tuple, default (1.3, 1) Bounding box anchor position for the legend.
Returns
matplotlib.figure.Figure Stacked bar plot visualization of composition data.
def
cell_regression( self, cell_x: str, cell_y: str, set_x: str | None = None, set_y: str | None = None, min_log=0.5, n_max=20, min_exp=0.1, min_pct=0.5, n_proc=10, min_esm=0.5, p_val=0.05, adj=True, image_width=12, image_high=7):
4319 def cell_regression( 4320 self, 4321 cell_x: str, 4322 cell_y: str, 4323 set_x: str | None = None, 4324 set_y: str | None = None, 4325 min_log=0.5, 4326 n_max=20, 4327 min_exp=0.1, 4328 min_pct=0.5, 4329 n_proc=10, 4330 min_esm=0.5, 4331 p_val=0.05, 4332 adj=True, 4333 image_width=12, 4334 image_high=7, 4335 ): 4336 """ 4337 Perform linear regression between two selected cell populations and identify 4338 significant and top-ranked differentially expressed genes. 4339 4340 The function compares the expression profiles of two selected cells or cell 4341 populations using linear regression. Each gene is represented as a point in 4342 the regression plot, with expression in `cell_x` on the X-axis and expression 4343 in `cell_y` on the Y-axis. 4344 4345 Differential expression statistics are calculated using `calc_DEG`. Genes are 4346 first filtered according to the minimum absolute log fold change, minimum 4347 absolute effect size (ESM), and statistical significance. Among the 4348 significant genes, up to `n_max` genes are selected within each `valid_group`, 4349 prioritizing genes with the largest standardized absolute regression residuals 4350 and absolute ESM values. 4351 4352 The regression plot uses three point categories: 4353 - gray: genes that are not statistically significant, 4354 - green: statistically significant genes that pass the DEG thresholds, 4355 - blue: top `n_max` genes selected within each `valid_group`. 4356 4357 The selected top genes are additionally annotated with their gene names. 4358 The plot also displays the fitted linear regression line, R-squared, 4359 p-value, and regression equation. 4360 4361 Parameters 4362 ---------- 4363 cell_x : str 4364 Name of the first cell or cell population used as the X-axis. 4365 4366 cell_y : str 4367 Name of the second cell or cell population used as the Y-axis. 4368 4369 set_x : str or None 4370 Dataset or experimental set identifier corresponding to `cell_x`. 4371 Required when multiple datasets contain the same cell name. If None, 4372 the cell is selected only by its name. 4373 4374 set_y : str or None 4375 Dataset or experimental set identifier corresponding to `cell_y`. 4376 Required when multiple datasets contain the same cell name. If None, 4377 the cell is selected only by its name. 4378 4379 min_log : float, default=0.5 4380 Minimum absolute log fold change required for a gene to be considered 4381 for significant DEG selection. 4382 4383 n_max : int, default=20 4384 Maximum number of top genes selected from each `valid_group`. 4385 Genes are ranked primarily by standardized absolute regression residual 4386 and secondarily by absolute Effect Size - Cohen's d. 4387 4388 min_exp : float, default=0.1 4389 Minimum expression threshold passed to `calc_DEG`. 4390 4391 min_pct : float, default=0.5 4392 Minimum fraction of cells in which a gene must be expressed, passed to 4393 `calc_DEG`. 4394 4395 n_proc : int, default=10 4396 Number of parallel processes used by `calc_DEG`. 4397 4398 min_esm : float, default=0.5 4399 Minimum absolute ESM (effect size measure) required for DEG selection. 4400 4401 p_val : float, default=0.05 4402 Maximum allowed p-value for statistical significance. If `adj=True`, 4403 this threshold is applied to the adjusted p-value (`adj_pval`); 4404 otherwise, it is applied to the raw p-value (`p_val`). 4405 4406 adj : bool, default=True 4407 Whether to use the adjusted p-value (`adj_pval`) instead of the raw 4408 p-value (`p_val`) for statistical significance filtering. 4409 4410 image_width : int or float, default=12 4411 Width of the regression plot in inches. 4412 4413 image_high : int or float, default=7 4414 Height of the regression plot in inches. 4415 4416 4417 4418 Returns 4419 ------- 4420 deg_stats : pandas.DataFrame 4421 DataFrame containing the DEG statistics after filtering by absolute 4422 log fold change, absolute ESM, and statistical significance. 4423 4424 fig : matplotlib.figure.Figure 4425 Matplotlib figure containing the regression plot, including expression 4426 points, regression line, regression statistics, and annotations for 4427 the selected top genes. 4428 4429 Raises 4430 ------ 4431 ValueError 4432 If `cell_x` or `cell_y` is not found in the dataset. 4433 4434 ValueError 4435 If multiple columns correspond to the same cell name and the required 4436 dataset identifiers (`set_x` and/or `set_y`) were not provided. 4437 4438 Notes 4439 ----- 4440 * The function calls `self.average()` before performing the analysis. 4441 * Linear regression is calculated using `scipy.stats.linregress`. 4442 * Regression residuals are calculated as the difference between the observed 4443 Y value and the value predicted by the fitted regression model. 4444 * Standardized residuals are used to prioritize genes that deviate most 4445 strongly from the expected linear relationship. 4446 * Significant genes are filtered using absolute log fold change, absolute 4447 ESM, and either raw or adjusted p-value. 4448 * Top genes are selected separately within each `valid_group`. 4449 * The top genes are annotated on the regression plot. 4450 * The returned `deg_stats` contains the significant genes before the final 4451 `n_max` per-group selection. 4452 4453 4454 Examples 4455 -------- 4456 > > > deg_stats, fig = obj.cell_regression( 4457 > > > ... cell_x="NeuronA", 4458 > > > ... cell_y="NeuronB", 4459 > > > ... min_log_min=0.5, 4460 > > > ... min_esm=0.5, 4461 > > > ... p_val=0.05, 4462 > > > ... n_max=20 4463 > > > ... ) 4464 > > > 4465 """ 4466 4467 self.average() 4468 4469 metadata = self.agg_metadata 4470 data = self.agg_normalized_data 4471 data2 = self.normalized_data 4472 metadata2 = self.input_metadata 4473 4474 if set_x is not None and set_y is not None: 4475 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4476 data2.columns = metadata2["cell_names"] + " # " + metadata2["sets"] 4477 cell_x = cell_x + " # " + set_x 4478 cell_y = cell_y + " # " + set_y 4479 4480 deg_stats = calc_DEG( 4481 data2, 4482 metadata_list=None, 4483 entities={ 4484 "cell_x": cell_x + " # " + set_x, 4485 "cell_y": cell_y + " # " + set_y, 4486 }, 4487 sets=None, 4488 min_exp=min_exp, 4489 min_pct=min_pct, 4490 n_proc=n_proc, 4491 ) 4492 4493 else: 4494 data.columns = metadata["cell_names"] 4495 data2.columns = metadata2["cell_names"] 4496 4497 deg_stats = calc_DEG( 4498 data2, 4499 metadata_list=None, 4500 entities={"cell_x": cell_x, "cell_y": cell_y}, 4501 sets=None, 4502 min_exp=min_exp, 4503 min_pct=min_pct, 4504 n_proc=n_proc, 4505 ) 4506 4507 # points 4508 x = data[cell_x] 4509 y = data[cell_y] 4510 4511 # regression 4512 slope, intercept, r_value, p_value, std_err = stats.linregress(x, y) 4513 4514 # prediction 4515 y_pred = slope * x + intercept 4516 4517 # residuals 4518 residual = y - y_pred 4519 4520 # residuals for DEG 4521 deg_stats["residual"] = deg_stats["feature"].map(residual) 4522 deg_stats["residual_abs"] = deg_stats["residual"].abs() 4523 deg_stats["residual_std"] = deg_stats["residual"] / residual.std() 4524 deg_stats["residual_std_abs"] = deg_stats["residual_std"].abs() 4525 deg_stats["esm_abs"] = deg_stats["esm"].abs() 4526 4527 deg_stats = deg_stats[deg_stats["log(FC)"].abs() >= min_log] 4528 4529 deg_stats = deg_stats[deg_stats["esm_abs"] >= min_esm] 4530 4531 if adj: 4532 4533 deg_stats = deg_stats[deg_stats["adj_pval"] <= p_val] 4534 else: 4535 deg_stats = deg_stats[deg_stats["p_val"] <= p_val] 4536 4537 deg_stats_n = ( 4538 deg_stats.sort_values( 4539 ["residual_std_abs", "esm_abs"], ascending=[False, False] 4540 ) 4541 .groupby("valid_group") 4542 .head(n_max) 4543 ) 4544 4545 if not cell_x in data.columns: 4546 raise ValueError("'cell_x' value not in cell names!") 4547 4548 if not cell_y in data.columns: 4549 raise ValueError("'cell_y' value not in cell names!") 4550 4551 if list(data.columns).count(cell_x) > 1: 4552 raise ValueError( 4553 f"'{cell_x}' occurs more than once. If you want to select a specific cell, " 4554 f"please also provide the corresponding 'set_x' and 'set_y' values." 4555 ) 4556 4557 if list(data.columns).count(cell_y) > 1: 4558 raise ValueError( 4559 f"'{cell_y}' occurs more than once. If you want to select a specific cell, " 4560 f"please also provide the corresponding 'set_x' and 'set_y' values." 4561 ) 4562 4563 data["color"] = "gray" 4564 4565 data.loc[data.index.isin(set(deg_stats["feature"])), "color"] = "green" 4566 4567 data.loc[data.index.isin(set(deg_stats_n["feature"])), "color"] = "blue" 4568 4569 color_order = {"gray": 0, "green": 1, "blue": 2} 4570 4571 data["color_order"] = data["color"].map(color_order) 4572 4573 data = data.sort_values("color_order") 4574 4575 fig, ax = plt.subplots(figsize=(image_width, image_high)) 4576 4577 # dots 4578 sns.scatterplot( 4579 x=cell_x, 4580 y=cell_y, 4581 data=data, 4582 hue="color", 4583 palette={"blue": "blue", "green": "green", "gray": "gray"}, 4584 legend=False, 4585 ax=ax, 4586 ) 4587 4588 # regression 4589 sns.regplot(x=cell_x, y=cell_y, data=data, scatter=False, color="red", ax=ax) 4590 4591 legend_elements = [ 4592 Line2D( 4593 [0], 4594 [0], 4595 marker="o", 4596 color="w", 4597 markerfacecolor="gray", 4598 markersize=7, 4599 label="Not significant", 4600 ), 4601 Line2D( 4602 [0], 4603 [0], 4604 marker="o", 4605 color="w", 4606 markerfacecolor="blue", 4607 markersize=7, 4608 label=f"Top {n_max}", 4609 ), 4610 Line2D( 4611 [0], 4612 [0], 4613 marker="o", 4614 color="w", 4615 markerfacecolor="green", 4616 markersize=7, 4617 label="Significant", 4618 ), 4619 Line2D([0], [0], color="red", linewidth=2, label="Linear regression"), 4620 ] 4621 4622 ax.legend(handles=legend_elements, loc="best", frameon=False) 4623 4624 slope, intercept, r_value, p_value, _ = stats.linregress( 4625 data[cell_x], data[cell_y] 4626 ) 4627 4628 equation = "y = {:.2f}x + {:.2f}".format(slope, intercept) 4629 4630 ax.annotate( 4631 "R-squared = {:.2f}\nP-value = {:.2f}\n{}".format( 4632 r_value**2, p_value, equation 4633 ), 4634 xy=(0.05, 0.90), 4635 xycoords="axes fraction", 4636 fontsize=12, 4637 ) 4638 4639 ax.spines["top"].set_visible(False) 4640 ax.spines["right"].set_visible(False) 4641 4642 # oznaczanie genów 4643 def annotate_outliers(x, y, features): 4644 texts = [] 4645 4646 for i, (xi, yi) in enumerate(zip(x, y)): 4647 if data.index[i] in features: 4648 text = ax.text(xi, yi, data.index[i]) 4649 texts.append(text) 4650 4651 return texts 4652 4653 features = list(set(deg_stats_n["feature"])) 4654 4655 texts = annotate_outliers(data[cell_x], data[cell_y], features) 4656 4657 adjust_text(texts, arrowprops=dict(arrowstyle="-", color="gray", alpha=0.5)) 4658 4659 return deg_stats, fig
Perform linear regression between two selected cell populations and identify significant and top-ranked differentially expressed genes.
The function compares the expression profiles of two selected cells or cell
populations using linear regression. Each gene is represented as a point in
the regression plot, with expression in cell_x on the X-axis and expression
in cell_y on the Y-axis.
Differential expression statistics are calculated using calc_DEG. Genes are
first filtered according to the minimum absolute log fold change, minimum
absolute effect size (ESM), and statistical significance. Among the
significant genes, up to n_max genes are selected within each valid_group,
prioritizing genes with the largest standardized absolute regression residuals
and absolute ESM values.
The regression plot uses three point categories:
- gray: genes that are not statistically significant,
- green: statistically significant genes that pass the DEG thresholds,
- blue: top
n_maxgenes selected within eachvalid_group.
The selected top genes are additionally annotated with their gene names. The plot also displays the fitted linear regression line, R-squared, p-value, and regression equation.
Parameters
cell_x : str Name of the first cell or cell population used as the X-axis.
cell_y : str Name of the second cell or cell population used as the Y-axis.
set_x : str or None
Dataset or experimental set identifier corresponding to cell_x.
Required when multiple datasets contain the same cell name. If None,
the cell is selected only by its name.
set_y : str or None
Dataset or experimental set identifier corresponding to cell_y.
Required when multiple datasets contain the same cell name. If None,
the cell is selected only by its name.
min_log : float, default=0.5 Minimum absolute log fold change required for a gene to be considered for significant DEG selection.
n_max : int, default=20
Maximum number of top genes selected from each valid_group.
Genes are ranked primarily by standardized absolute regression residual
and secondarily by absolute Effect Size - Cohen's d.
min_exp : float, default=0.1
Minimum expression threshold passed to calc_DEG.
min_pct : float, default=0.5
Minimum fraction of cells in which a gene must be expressed, passed to
calc_DEG.
n_proc : int, default=10
Number of parallel processes used by calc_DEG.
min_esm : float, default=0.5 Minimum absolute ESM (effect size measure) required for DEG selection.
p_val : float, default=0.05
Maximum allowed p-value for statistical significance. If adj=True,
this threshold is applied to the adjusted p-value (adj_pval);
otherwise, it is applied to the raw p-value (p_val).
adj : bool, default=True
Whether to use the adjusted p-value (adj_pval) instead of the raw
p-value (p_val) for statistical significance filtering.
image_width : int or float, default=12 Width of the regression plot in inches.
image_high : int or float, default=7 Height of the regression plot in inches.
Returns
deg_stats : pandas.DataFrame DataFrame containing the DEG statistics after filtering by absolute log fold change, absolute ESM, and statistical significance.
fig : matplotlib.figure.Figure Matplotlib figure containing the regression plot, including expression points, regression line, regression statistics, and annotations for the selected top genes.
Raises
ValueError
If cell_x or cell_y is not found in the dataset.
ValueError
If multiple columns correspond to the same cell name and the required
dataset identifiers (set_x and/or set_y) were not provided.
Notes
- The function calls
self.average()before performing the analysis. - Linear regression is calculated using
scipy.stats.linregress. - Regression residuals are calculated as the difference between the observed Y value and the value predicted by the fitted regression model.
- Standardized residuals are used to prioritize genes that deviate most strongly from the expected linear relationship.
- Significant genes are filtered using absolute log fold change, absolute ESM, and either raw or adjusted p-value.
- Top genes are selected separately within each
valid_group. - The top genes are annotated on the regression plot.
- The returned
deg_statscontains the significant genes before the finaln_maxper-group selection.
Examples
deg_stats, fig = obj.cell_regression( ... cell_x="NeuronA", ... cell_y="NeuronB", ... min_log_min=0.5, ... min_esm=0.5, ... p_val=0.05, ... n_max=20 ... )
def
cell_genes_distribution( self, cells_names: list | None, cells_sets: list | None = None, image_width=10, image_high=7):
4661 def cell_genes_distribution( 4662 self, 4663 cells_names: list | None, 4664 cells_sets: list | None = None, 4665 image_width=10, 4666 image_high=7, 4667 ): 4668 """ 4669 Compare the distributions of the number of detected genes per cell across 4670 selected cell populations. 4671 4672 For each selected cell population, the function calculates the number of 4673 detected genes in each cell, defined as the number of genes with an expression 4674 value greater than zero. The resulting distributions are visualized as 4675 overlapping histograms, allowing the gene detection levels of different cell 4676 populations to be compared. 4677 4678 This analysis can be used as a quality-control step before comparing gene 4679 expression profiles or performing differential expression analysis. Similar 4680 distributions of detected genes per cell indicate comparable levels of gene 4681 detection across the selected populations, whereas substantial differences 4682 may indicate differences in sequencing depth, RNA content, or other technical 4683 factors. 4684 4685 Parameters 4686 ---------- 4687 cells_names : list 4688 List of cell population or cluster names to be included in the analysis. 4689 4690 cells_sets : list or None 4691 Optional list of dataset or experimental set identifiers corresponding 4692 element-wise to `cells_names`. This is required when the same cell 4693 population name occurs in multiple datasets and a specific dataset 4694 needs to be selected. 4695 4696 image_width : int or float, default=10 4697 Width of the histogram figure in inches. 4698 4699 image_high : int or float, default=7 4700 Height of the histogram figure in inches. 4701 4702 Returns 4703 ------- 4704 fig : matplotlib.figure.Figure 4705 Matplotlib figure containing overlapping histograms showing the 4706 distribution of the number of detected genes per cell for each selected 4707 cell population. 4708 4709 Raises 4710 ------ 4711 ValueError 4712 If `cells_names` is None or empty. 4713 4714 Notes 4715 ----- 4716 * A gene is considered detected when its expression value is greater than zero. 4717 * The number of detected genes is calculated independently for every cell. 4718 * Each selected cell population is represented by a separate histogram. 4719 * When `cells_sets` is provided and has the same length as `cells_names`, 4720 cell populations are selected using both their population name and dataset 4721 identifier. 4722 * The function operates on `self.normalized_data`. 4723 * The resulting distributions can be used to assess whether selected cell 4724 populations have comparable gene-detection levels before downstream 4725 comparisons. 4726 """ 4727 4728 metadata = self.input_metadata 4729 data = self.normalized_data 4730 4731 if cells_names is None or len(cells_names) == 0: 4732 raise ValueError("No cells selected!") 4733 4734 if cells_sets is not None and len(cells_sets) == len(cells_names): 4735 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4736 full_cells = [] 4737 for n in range(len(cells_names)): 4738 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4739 4740 data = data.loc[:, full_cells] 4741 4742 else: 4743 data.columns = metadata["cell_names"] 4744 data = data.loc[:, cells_names] 4745 4746 fig = plt.figure(figsize=(image_width, image_high)) 4747 4748 for name in data.columns.unique(): 4749 cols = data.columns == name 4750 4751 counts = (data.loc[:, cols] > 0).sum(axis=0) 4752 4753 plt.hist(counts, bins=30, alpha=0.5, label=name) 4754 4755 plt.xlabel("Number of detected genes per cell") 4756 plt.ylabel("Number of cells") 4757 plt.legend() 4758 plt.tight_layout() 4759 4760 return fig
Compare the distributions of the number of detected genes per cell across selected cell populations.
For each selected cell population, the function calculates the number of detected genes in each cell, defined as the number of genes with an expression value greater than zero. The resulting distributions are visualized as overlapping histograms, allowing the gene detection levels of different cell populations to be compared.
This analysis can be used as a quality-control step before comparing gene expression profiles or performing differential expression analysis. Similar distributions of detected genes per cell indicate comparable levels of gene detection across the selected populations, whereas substantial differences may indicate differences in sequencing depth, RNA content, or other technical factors.
Parameters
cells_names : list List of cell population or cluster names to be included in the analysis.
cells_sets : list or None
Optional list of dataset or experimental set identifiers corresponding
element-wise to cells_names. This is required when the same cell
population name occurs in multiple datasets and a specific dataset
needs to be selected.
image_width : int or float, default=10 Width of the histogram figure in inches.
image_high : int or float, default=7 Height of the histogram figure in inches.
Returns
fig : matplotlib.figure.Figure Matplotlib figure containing overlapping histograms showing the distribution of the number of detected genes per cell for each selected cell population.
Raises
ValueError
If cells_names is None or empty.
Notes
- A gene is considered detected when its expression value is greater than zero.
- The number of detected genes is calculated independently for every cell.
- Each selected cell population is represented by a separate histogram.
- When
cells_setsis provided and has the same length ascells_names, cell populations are selected using both their population name and dataset identifier. - The function operates on
self.normalized_data. - The resulting distributions can be used to assess whether selected cell populations have comparable gene-detection levels before downstream comparisons.
def
mm_cell_genes( self, cells_names: list | None, cells_sets: list | None, non_include_zeros: bool = True, image_width=7, image_high=10):
4762 def mm_cell_genes( 4763 self, 4764 cells_names: list | None, 4765 cells_sets: list | None, 4766 non_include_zeros: bool = True, 4767 image_width=7, 4768 image_high=10, 4769 ): 4770 """ 4771 Compare the mean and median gene expression across selected cell populations. 4772 4773 For each selected cell population, the function calculates the mean and median 4774 expression values across all genes and cells. By default, zero-expression 4775 values are excluded from the calculation, allowing the comparison to focus on 4776 genes with detected expression. When `non_include_zeros` is set to False, zero 4777 values are included in the calculation. 4778 4779 The resulting mean and median expression values are visualized as a grouped 4780 bar plot, with separate bars representing the mean and median for each cell 4781 population. This visualization can be used to assess differences in the overall 4782 expression level and distribution between selected cell populations. 4783 4784 Parameters 4785 ---------- 4786 cells_names : list 4787 List of cell population or cluster names to be included in the analysis. 4788 4789 cells_sets : list or None 4790 Optional list of dataset or experimental set identifiers corresponding 4791 element-wise to `cells_names`. When provided and its length matches 4792 `cells_names`, cell populations are selected using both their population 4793 name and dataset identifier. 4794 4795 non_include_zeros : bool, default=True 4796 Whether to exclude zero-expression values before calculating the mean and 4797 median. If True, only non-zero expression values are used. If False, all 4798 expression values, including zeros, are used. 4799 4800 image_width : int or float, default=7 4801 Width of the figure in inches. 4802 4803 image_high : int or float, default=10 4804 Height of the figure in inches. 4805 4806 Returns 4807 ------- 4808 fig : matplotlib.figure.Figure 4809 Matplotlib figure containing grouped bar plots showing the mean and median 4810 expression for each selected cell population. 4811 4812 Raises 4813 ------ 4814 ValueError 4815 If `cells_names` is None or empty. 4816 4817 Notes 4818 ----- 4819 * Expression values are obtained from `self.normalized_data`. 4820 * The mean and median are calculated independently for each selected cell 4821 population. 4822 * When `non_include_zeros=True`, zero-expression values are excluded before 4823 calculating both statistics. 4824 * When `non_include_zeros=False`, zero-expression values contribute to both 4825 the mean and median. 4826 * The analysis uses all genes and cells belonging to each selected population. 4827 * The resulting statistics describe the overall distribution of expression 4828 values and can be used to compare the expression levels of different cell 4829 populations. 4830 """ 4831 4832 metadata = self.input_metadata 4833 data = self.normalized_data 4834 4835 if cells_names is None or len(cells_names) == 0: 4836 raise ValueError("No cells selected!") 4837 4838 if cells_sets is not None and len(cells_sets) == len(cells_names): 4839 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4840 full_cells = [] 4841 for n in range(len(cells_names)): 4842 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4843 4844 data = data.loc[:, full_cells] 4845 4846 else: 4847 data.columns = metadata["cell_names"] 4848 data = data.loc[:, cells_names] 4849 4850 groups = data.columns.unique() 4851 4852 fig = plt.figure(figsize=(image_width, image_high)) 4853 4854 mean_expression = {} 4855 median_expression = {} 4856 4857 for name in groups: 4858 cols = data.columns == name 4859 4860 values = data.loc[:, cols].values.flatten() 4861 4862 if non_include_zeros: 4863 values = values[values != 0] 4864 4865 mean_expression[name] = values.mean() 4866 median_expression[name] = np.median(values) 4867 4868 mean_expression = pd.Series(mean_expression) 4869 median_expression = pd.Series(median_expression) 4870 4871 x = np.arange(len(groups)) 4872 width = 0.35 4873 4874 plt.bar(x - width / 2, mean_expression.values, width, label="Mean") 4875 4876 plt.bar(x + width / 2, median_expression.values, width, label="Median") 4877 4878 plt.xlabel("Cell") 4879 plt.ylabel("Expression") 4880 plt.xticks(x, groups, rotation=45) 4881 plt.legend() 4882 4883 plt.tight_layout() 4884 4885 return fig
Compare the mean and median gene expression across selected cell populations.
For each selected cell population, the function calculates the mean and median
expression values across all genes and cells. By default, zero-expression
values are excluded from the calculation, allowing the comparison to focus on
genes with detected expression. When non_include_zeros is set to False, zero
values are included in the calculation.
The resulting mean and median expression values are visualized as a grouped bar plot, with separate bars representing the mean and median for each cell population. This visualization can be used to assess differences in the overall expression level and distribution between selected cell populations.
Parameters
cells_names : list List of cell population or cluster names to be included in the analysis.
cells_sets : list or None
Optional list of dataset or experimental set identifiers corresponding
element-wise to cells_names. When provided and its length matches
cells_names, cell populations are selected using both their population
name and dataset identifier.
non_include_zeros : bool, default=True Whether to exclude zero-expression values before calculating the mean and median. If True, only non-zero expression values are used. If False, all expression values, including zeros, are used.
image_width : int or float, default=7 Width of the figure in inches.
image_high : int or float, default=10 Height of the figure in inches.
Returns
fig : matplotlib.figure.Figure Matplotlib figure containing grouped bar plots showing the mean and median expression for each selected cell population.
Raises
ValueError
If cells_names is None or empty.
Notes
- Expression values are obtained from
self.normalized_data. - The mean and median are calculated independently for each selected cell population.
- When
non_include_zeros=True, zero-expression values are excluded before calculating both statistics. - When
non_include_zeros=False, zero-expression values contribute to both the mean and median. - The analysis uses all genes and cells belonging to each selected population.
- The resulting statistics describe the overall distribution of expression values and can be used to compare the expression levels of different cell populations.
def
cell_dispersion( self, cells_names: list | None, cells_sets: list | None, image_width=7, image_high=10):
4887 def cell_dispersion( 4888 self, 4889 cells_names: list | None, 4890 cells_sets: list | None, 4891 image_width=7, 4892 image_high=10, 4893 ): 4894 """ 4895 Assess the within-cluster dispersion of cell gene expression profiles. 4896 4897 For each selected cell population, the function calculates the centroid of 4898 the population as the mean expression profile across all cells in the cluster. 4899 The Euclidean distance between each individual cell and its corresponding 4900 cluster centroid is then calculated to quantify how much the cell's expression 4901 profile deviates from the average profile of the cluster. 4902 4903 The resulting distributions of distances are visualized using violin plots, 4904 with each violin representing one selected cell population. The distribution 4905 shows the degree of transcriptional heterogeneity within each cluster: 4906 smaller distances indicate cells with more similar expression profiles, 4907 whereas larger distances indicate greater within-cluster variability. 4908 4909 This analysis can be used to assess the internal consistency and transcriptional 4910 heterogeneity of cell populations before comparing their expression profiles 4911 between clusters. 4912 4913 Parameters 4914 ---------- 4915 self : object 4916 self object containing the input metadata and normalized expression 4917 data. 4918 4919 cells_names : list 4920 List of cell population or cluster names to be included in the analysis. 4921 4922 cells_sets : list or None 4923 Optional list of dataset or experimental set identifiers corresponding 4924 element-wise to `cells_names`. When provided and its length matches 4925 `cells_names`, cell populations are selected using both their population 4926 name and dataset identifier. 4927 4928 image_width : int or float, default=7 4929 Width of the figure in inches. 4930 4931 image_high : int or float, default=10 4932 Height of the figure in inches. 4933 4934 Returns 4935 ------- 4936 fig : matplotlib.figure.Figure 4937 Matplotlib figure containing violin plots showing the distribution of 4938 Euclidean distances between individual cells and their respective cluster 4939 centroids. 4940 4941 Raises 4942 ------ 4943 ValueError 4944 If `cells_names` is None or empty. 4945 4946 Notes 4947 ----- 4948 * Expression values are obtained from `self.normalized_data`. 4949 * A cluster centroid is calculated as the mean expression profile across all 4950 cells belonging to that cluster. 4951 * The distance of each cell from the centroid is calculated using Euclidean 4952 distance across all genes. 4953 * Each cell contributes one distance value to the distribution of its cluster. 4954 * Smaller distances indicate lower within-cluster transcriptional dispersion. 4955 * Larger distances indicate greater within-cluster transcriptional dispersion. 4956 * The violin plot includes a box plot showing the median and interquartile 4957 range of the distance distribution. 4958 * Differences in dispersion between clusters can be used to assess their 4959 relative transcriptional homogeneity before downstream comparisons. 4960 """ 4961 4962 metadata = self.input_metadata 4963 data = self.normalized_data 4964 4965 if cells_names is None or len(cells_names) == 0: 4966 raise ValueError("No cells selected!") 4967 4968 if cells_sets is not None and len(cells_sets) == len(cells_names): 4969 data.columns = metadata["cell_names"] + " # " + metadata["sets"] 4970 full_cells = [] 4971 for n in range(len(cells_names)): 4972 full_cells.append(cells_names[n] + " # " + cells_sets[n]) 4973 4974 data = data.loc[:, full_cells] 4975 4976 else: 4977 data.columns = metadata["cell_names"] 4978 data = data.loc[:, cells_names] 4979 4980 fig = plt.figure(figsize=(image_width, image_high)) 4981 4982 dispersion_long = [] 4983 4984 for cluster in data.columns.unique(): 4985 4986 cells = data.loc[:, data.columns == cluster] 4987 4988 centroid = cells.mean(axis=1) 4989 4990 distances = np.sqrt(((cells.sub(centroid, axis=0)) ** 2).sum(axis=0)) 4991 4992 for cell, distance in distances.items(): 4993 dispersion_long.append( 4994 {"cluster": cluster, "cell": cell, "distance_to_centroid": distance} 4995 ) 4996 4997 dispersion_long = pd.DataFrame(dispersion_long) 4998 4999 sns.violinplot( 5000 data=dispersion_long, x="cluster", y="distance_to_centroid", inner="box" 5001 ) 5002 5003 plt.ylabel("Distance from cluster centroid") 5004 plt.xlabel("Cluster") 5005 plt.xticks(rotation=45) 5006 plt.tight_layout() 5007 5008 return fig
Assess the within-cluster dispersion of cell gene expression profiles.
For each selected cell population, the function calculates the centroid of the population as the mean expression profile across all cells in the cluster. The Euclidean distance between each individual cell and its corresponding cluster centroid is then calculated to quantify how much the cell's expression profile deviates from the average profile of the cluster.
The resulting distributions of distances are visualized using violin plots, with each violin representing one selected cell population. The distribution shows the degree of transcriptional heterogeneity within each cluster: smaller distances indicate cells with more similar expression profiles, whereas larger distances indicate greater within-cluster variability.
This analysis can be used to assess the internal consistency and transcriptional heterogeneity of cell populations before comparing their expression profiles between clusters.
Parameters
self : object self object containing the input metadata and normalized expression data.
cells_names : list List of cell population or cluster names to be included in the analysis.
cells_sets : list or None
Optional list of dataset or experimental set identifiers corresponding
element-wise to cells_names. When provided and its length matches
cells_names, cell populations are selected using both their population
name and dataset identifier.
image_width : int or float, default=7 Width of the figure in inches.
image_high : int or float, default=10 Height of the figure in inches.
Returns
fig : matplotlib.figure.Figure Matplotlib figure containing violin plots showing the distribution of Euclidean distances between individual cells and their respective cluster centroids.
Raises
ValueError
If cells_names is None or empty.
Notes
- Expression values are obtained from
self.normalized_data. - A cluster centroid is calculated as the mean expression profile across all cells belonging to that cluster.
- The distance of each cell from the centroid is calculated using Euclidean distance across all genes.
- Each cell contributes one distance value to the distribution of its cluster.
- Smaller distances indicate lower within-cluster transcriptional dispersion.
- Larger distances indicate greater within-cluster transcriptional dispersion.
- The violin plot includes a box plot showing the median and interquartile range of the distance distribution.
- Differences in dispersion between clusters can be used to assess their relative transcriptional homogeneity before downstream comparisons.