CORPUS_ANALYSIS.PY

def analyze_corpus_patterns(self, corpus_data: List[Dict[str, Any]]) -> Dict[str, Any]: """Analyze patterns across entire corpus""" # Extract HRV vectors from corpus hrv_vectors = [doc["hrv_vector"] for doc in corpus_data] hrv_matrix = np.array(hrv_vectors) # Statistical analysis analysis = { "corpus_statistics": { "total_documents": len(corpus_data), "avg_hrv_score": np.mean(hrv_matrix.mean(axis=1)), "hrv_std_dev": np.std(hrv_matrix), "dimension_means": np.mean(hrv_matrix, axis=0).tolist(), "dimension_std_devs": np.std(hrv_matrix, axis=0).tolist() }, "quality_distribution": self._calculate_quality_distribution(corpus_data), "style_clusters": self._identify_style_clusters(hrv_matrix), "outlier_analysis": self._detect_outliers(hrv_matrix), "correlation_analysis": self._analyze_dimension_correlations(hrv_matrix) } return analysis def _identify_style_clusters(self, hrv_matrix: np.ndarray) -> Dict[str, Any]: """Identify writing style clusters using HRV patterns""" # Simple clustering based on HRV patterns from sklearn.cluster import KMeans # Cluster documents into style groups kmeans = KMeans(n_clusters=3, random_state=42) cluster_labels = kmeans.fit_predict(hrv_matrix) # Analyze cluster characteristics clusters = {} for cluster_id in range(3): cluster_mask = cluster_labels == cluster_id cluster_hrv = hrv_matrix[cluster_mask] clusters[f"cluster_{cluster_id}"] = { "size": len(cluster_hrv), "avg_hrv": np.mean(cluster_hrv, axis=0).tolist(), "dominant_style": self._classify_cluster_style(np.mean(cluster_hrv, axis=0)) } return clusters