From 490e355a1ad81bcdf8655d0b9fe62e4a9c389d32 Mon Sep 17 00:00:00 2001 From: Alejandro Ponce Date: Wed, 5 Mar 2025 13:17:21 +0200 Subject: [PATCH] Validate persona description is sufficiently different Closes: #1218 Check if the description for a new persona is different enough from the existing personas descriptions. This is done to correctly differentiate between personas --- src/codegate/config.py | 5 + src/codegate/db/connection.py | 20 + src/codegate/db/models.py | 2 + src/codegate/muxing/semantic_router.py | 31 ++ tests/muxing/test_semantic_router.py | 596 ++++++++----------------- 5 files changed, 250 insertions(+), 404 deletions(-) diff --git a/src/codegate/config.py b/src/codegate/config.py index 761ca09e..179ec4d3 100644 --- a/src/codegate/config.py +++ b/src/codegate/config.py @@ -57,9 +57,14 @@ class Config: force_certs: bool = False max_fim_hash_lifetime: int = 60 * 5 # Time in seconds. Default is 5 minutes. + # Min value is 0 (max similarity), max value is 2 (orthogonal) # The value 0.75 was found through experimentation. See /tests/muxing/test_semantic_router.py + # It's the threshold value to determine if a query matches a persona. persona_threshold = 0.75 + # The value 0.3 was found through experimentation. See /tests/muxing/test_semantic_router.py + # It's the threshold value to determine if a persona description is similar to existing personas + persona_diff_desc_threshold = 0.3 # Provider URLs with defaults provider_urls: Dict[str, str] = field(default_factory=lambda: DEFAULT_PROVIDER_URLS.copy()) diff --git a/src/codegate/db/connection.py b/src/codegate/db/connection.py index 170cb52e..420f27e8 100644 --- a/src/codegate/db/connection.py +++ b/src/codegate/db/connection.py @@ -1004,6 +1004,26 @@ async def get_persona_by_name(self, persona_name: str) -> Optional[Persona]: ) return personas[0] if personas else None + async def get_distance_to_existing_personas( + self, query_embedding: np.ndarray + ) -> List[PersonaDistance]: + """ + Get the distance between a persona and a query embedding. + """ + sql = """ + SELECT + id, + name, + description, + vec_distance_cosine(description_embedding, :query_embedding) as distance + FROM personas + """ + conditions = {"query_embedding": query_embedding} + persona_distances = await self._exec_vec_db_query_to_pydantic( + sql, conditions, PersonaDistance + ) + return persona_distances + async def get_distance_to_persona( self, persona_id: str, query_embedding: np.ndarray ) -> PersonaDistance: diff --git a/src/codegate/db/models.py b/src/codegate/db/models.py index a5941e96..f71e3c62 100644 --- a/src/codegate/db/models.py +++ b/src/codegate/db/models.py @@ -245,6 +245,8 @@ class MuxRule(BaseModel): def nd_array_custom_before_validator(x): # custome before validation logic + if isinstance(x, bytes): + return np.frombuffer(x, dtype=np.float32) return x diff --git a/src/codegate/muxing/semantic_router.py b/src/codegate/muxing/semantic_router.py index ce240b1f..27a25754 100644 --- a/src/codegate/muxing/semantic_router.py +++ b/src/codegate/muxing/semantic_router.py @@ -28,6 +28,10 @@ class PersonaDoesNotExistError(Exception): pass +class PersonaSimilarDescriptionError(Exception): + pass + + class SemanticRouter: def __init__(self): @@ -36,6 +40,7 @@ def __init__(self): self._embeddings_model = f"{conf.model_base_path}/{conf.embedding_model}" self._n_gpu = conf.chat_model_n_gpu_layers self._persona_threshold = conf.persona_threshold + self._persona_diff_desc_threshold = conf.persona_diff_desc_threshold self._db_recorder = DbRecorder() self._db_reader = DbReader() @@ -105,12 +110,38 @@ async def _embed_text(self, text: str) -> np.ndarray: logger.debug("Text embedded in semantic routing", text=cleaned_text[:50]) return np.array(embed_list[0], dtype=np.float32) + async def _is_persona_description_diff(self, emb_persona_desc: np.ndarray) -> bool: + """ + Check if the persona description is different enough from existing personas. + """ + # The distance calculation is done in the database + persona_distances = await self._db_reader.get_distance_to_existing_personas( + emb_persona_desc + ) + if not persona_distances: + return True + + for persona_distance in persona_distances: + logger.info( + f"Persona description distance to {persona_distance.name}", + distance=persona_distance.distance, + ) + # If the distance is less than the threshold, the persona description is too similar + if persona_distance.distance < self._persona_diff_desc_threshold: + return False + return True + async def add_persona(self, persona_name: str, persona_desc: str) -> None: """ Add a new persona to the database. The persona description is embedded and stored in the database. """ emb_persona_desc = await self._embed_text(persona_desc) + if not await self._is_persona_description_diff(emb_persona_desc): + raise PersonaSimilarDescriptionError( + "The persona description is too similar to existing personas." + ) + new_persona = db_models.PersonaEmbedding( id=str(uuid.uuid4()), name=persona_name, diff --git a/tests/muxing/test_semantic_router.py b/tests/muxing/test_semantic_router.py index c8c7edc6..57687567 100644 --- a/tests/muxing/test_semantic_router.py +++ b/tests/muxing/test_semantic_router.py @@ -6,7 +6,11 @@ from pydantic import BaseModel from codegate.db import connection -from codegate.muxing.semantic_router import PersonaDoesNotExistError, SemanticRouter +from codegate.muxing.semantic_router import ( + PersonaDoesNotExistError, + PersonaSimilarDescriptionError, + SemanticRouter, +) @pytest.fixture @@ -55,6 +59,19 @@ async def test_add_persona(semantic_router_mocked_db: SemanticRouter): assert retrieved_persona.description == persona_desc +@pytest.mark.asyncio +async def test_add_duplicate_persona(semantic_router_mocked_db: SemanticRouter): + """Test adding a persona to the database.""" + persona_name = "test_persona" + persona_desc = "test_persona_desc" + await semantic_router_mocked_db.add_persona(persona_name, persona_desc) + + # Update the description to not trigger the similarity check + updated_description = "foo and bar description" + with pytest.raises(connection.AlreadyExistsError): + await semantic_router_mocked_db.add_persona(persona_name, updated_description) + + @pytest.mark.asyncio async def test_persona_not_exist_match(semantic_router_mocked_db: SemanticRouter): """Test checking persona match when persona does not exist""" @@ -78,475 +95,206 @@ class PersonaMatchTest(BaseModel): fail_queries=["foo"], ) -software_architect = PersonaMatchTest( - persona_name="software architect", +# Architect Persona +architect = PersonaMatchTest( + persona_name="architect", persona_desc=""" - Expert in designing large-scale software systems and technical infrastructure. - Specializes in distributed systems, microservices architecture, - and cloud-native applications. - Deep knowledge of architectural patterns like CQRS, event sourcing, hexagonal architecture, - and domain-driven design. - Experienced in designing scalable, resilient, and maintainable software solutions. - Proficient in evaluating technology stacks and making strategic technical decisions. - Skilled at creating architecture diagrams, technical specifications, - and system documentation. - Focuses on non-functional requirements like performance, security, and reliability. - Guides development teams on best practices for implementing complex systems. + Expert in designing and planning software systems, technical infrastructure, and solution + architecture. + Specializes in creating scalable, maintainable, and resilient system designs. + Deep knowledge of architectural patterns, principles, and best practices. + Experienced in evaluating technology stacks and making strategic technical decisions. + Skilled at creating architecture diagrams, technical specifications, and system + documentation. + Focuses on both functional and non-functional requirements like performance, security, + and reliability. + Guides development teams on implementing complex systems and following architectural + guidelines. + + Designs system architectures that balance business needs with technical constraints. + Creates technical roadmaps and migration strategies for legacy system modernization. + Evaluates trade-offs between different architectural approaches (monolithic, microservices, + serverless). + Implements domain-driven design principles to align software with business domains. + + Develops reference architectures and technical standards for organization-wide adoption. + Conducts architecture reviews and provides recommendations for improvement. + Collaborates with stakeholders to translate business requirements into technical solutions. + Stays current with emerging technologies and evaluates their potential application. + + Designs for cloud-native environments using containerization, orchestration, and managed + services. + Implements event-driven architectures using message queues, event buses, and streaming + platforms. + Creates data architectures that address storage, processing, and analytics requirements. + Develops integration strategies for connecting disparate systems and services. """, pass_queries=[ """ - How should I design a microservices architecture that can handle high traffic loads? + How should I design a system architecture that can scale with our growing user base? """, """ - What's the best approach for implementing event sourcing in a distributed system? + What's the best approach for migrating our monolithic application to microservices? """, """ - I need to design a system that can scale to millions of users. What architecture would you - recommend? + I need to create a technical roadmap for modernizing our legacy systems. Where should + I start? """, """ - Can you explain the trade-offs between monolithic and microservices architectures for our - new project? - """, - ], - fail_queries=[ - """ - How do I create a simple landing page with HTML and CSS? - """, - """ - What's the best way to optimize my SQL query performance? + Can you help me evaluate different cloud providers for our new infrastructure? """, """ - Can you help me debug this JavaScript function that's throwing an error? - """, - """ - How do I implement user authentication in my React application? - """, - ], -) - -# Data Scientist Persona -data_scientist = PersonaMatchTest( - persona_name="data scientist", - persona_desc=""" - Expert in analyzing and interpreting complex data to solve business problems. - Specializes in statistical analysis, machine learning algorithms, and predictive modeling. - Builds and deploys models for classification, regression, clustering, and anomaly detection. - Proficient in data preprocessing, feature engineering, and model evaluation techniques. - Uses Python with libraries like NumPy, Pandas, scikit-learn, TensorFlow, and PyTorch. - Experienced with data visualization using Matplotlib, Seaborn, and interactive dashboards. - Applies experimental design principles and A/B testing methodologies. - Works with structured and unstructured data, including time series and text. - Implements data pipelines for model training, validation, and deployment. - Communicates insights and recommendations based on data analysis to stakeholders. - - Handles class imbalance problems in classification tasks using techniques like SMOTE, - undersampling, oversampling, and class weighting. Addresses customer churn prediction - challenges by identifying key features that indicate potential churners. - - Applies feature selection methods for high-dimensional datasets, including filter methods - (correlation, chi-square), wrapper methods (recursive feature elimination), and embedded - methods (LASSO regularization). - - Prevents overfitting and high variance in tree-based models like random forests through - techniques such as pruning, setting maximum depth, adjusting minimum samples per leaf, - and cross-validation. - - Specializes in time series forecasting for sales and demand prediction, using methods like - ARIMA, SARIMA, Prophet, and exponential smoothing to handle seasonal patterns and trends. - Implements forecasting models that account for quarterly business cycles and seasonal - variations in customer behavior. - - Evaluates model performance using appropriate metrics: accuracy, precision, recall, - F1-score - for classification; RMSE, MAE, R-squared for regression; and specialized metrics for - time series forecasting like MAPE and SMAPE. - - Experienced in developing customer segmentation models, recommendation systems, - anomaly detection algorithms, and predictive maintenance solutions. - """, - pass_queries=[ - """ - How should I handle class imbalance in my customer churn prediction model? - """, - """ - What feature selection techniques would work best for my high-dimensional dataset? - """, - """ - I'm getting high variance in my random forest model. How can I prevent overfitting? - """, - """ - What's the best approach for forecasting seasonal time series data for our sales - predictions? + What architectural patterns would you recommend for a distributed e-commerce platform? """, ], fail_queries=[ """ - How do I structure my React components for a single-page application? + How do I fix this specific bug in my JavaScript code? """, """ - What's the best way to implement a CI/CD pipeline for my microservices? + What's the syntax for a complex SQL query joining multiple tables? """, """ - Can you help me design a responsive layout for mobile and desktop browsers? + How do I implement authentication in my React application? """, """ - How should I configure my Kubernetes cluster for high availability? + What's the best way to optimize the performance of this specific function? """, ], ) -# UX Designer Persona -ux_designer = PersonaMatchTest( - persona_name="ux designer", +# Coder Persona +coder = PersonaMatchTest( + persona_name="coder", persona_desc=""" - Expert in creating intuitive, user-centered digital experiences and interfaces. - Specializes in user research, usability testing, and interaction design. - Creates wireframes, prototypes, and user flows to visualize design solutions. - Conducts user interviews, usability studies, and analyzes user feedback. - Develops user personas and journey maps to understand user needs and pain points. - Designs information architecture and navigation systems for complex applications. - Applies design thinking methodology to solve user experience problems. - Knowledgeable about accessibility standards and inclusive design principles. - Collaborates with product managers and developers to implement user-friendly features. - Uses tools like Figma, Sketch, and Adobe XD to create high-fidelity mockups. + Expert in full stack development, programming, and software implementation. + Specializes in writing, debugging, and optimizing code across the entire technology stack. + + Proficient in multiple programming languages including JavaScript, Python, Java, C#, and + TypeScript. + Implements efficient algorithms and data structures to solve complex programming challenges. + Develops maintainable code with appropriate patterns and practices for different contexts. + + Experienced in frontend development using modern frameworks and libraries. + Creates responsive, accessible user interfaces with HTML, CSS, and JavaScript frameworks. + Implements state management, component architecture, + and client-side performance optimization for frontend applications. + + Skilled in backend development and server-side programming. + Builds RESTful APIs, GraphQL services, and microservices architectures. + Implements authentication, authorization, and security best practices in web applications. + Understands best ways for different backend problems, like file uploads, caching, + and database interactions. + + Designs and manages databases including schema design, query optimization, + and data modeling. + Works with both SQL and NoSQL databases to implement efficient data storage solutions. + Creates data access layers and ORM implementations for application data requirements. + + Handles integration between different systems and third-party services. + Implements webhooks, API clients, and service communication patterns. + Develops data transformation and processing pipelines for various application needs. + + Identifies and resolves performance issues across the application stack. + Uses debugging tools, profilers, and testing frameworks to ensure code quality. + Implements comprehensive testing strategies including unit, integration, + and end-to-end tests. """, pass_queries=[ """ - How can I improve the user onboarding experience for my mobile application? - """, - """ - What usability testing methods would you recommend for evaluating our new interface design? + How do I implement authentication in my web application? """, """ - I'm designing a complex dashboard. What information architecture would make it most - intuitive for users? + What's the best way to structure a RESTful API for my project? """, """ - How should I structure user research to identify pain points in our current - checkout process? - """, - ], - fail_queries=[ - """ - How do I configure a load balancer for my web servers? + I need help optimizing my database queries for better performance. """, """ - What's the best way to implement a caching layer in my application? + How should I implement state management in my frontend application? """, """ - Can you explain how to set up a CI/CD pipeline with GitHub Actions? - """, - """ - How do I optimize my database queries for better performance? - """, - ], -) - -# DevOps Engineer Persona -devops_engineer = PersonaMatchTest( - persona_name="devops engineer", - persona_desc=""" - Expertise: Infrastructure automation, CI/CD pipelines, cloud services, containerization, - and monitoring. - Proficient with tools like Docker, Kubernetes, Terraform, Ansible, and Jenkins. - Experienced with cloud platforms including AWS, Azure, and Google Cloud. - Strong knowledge of Linux/Unix systems administration and shell scripting. - Skilled in implementing microservices architectures and service mesh technologies. - Focus on reliability, scalability, security, and operational efficiency. - Practices infrastructure as code, GitOps, and site reliability engineering principles. - Experienced with monitoring tools like Prometheus, Grafana, and ELK stack. - """, - pass_queries=[ - """ - What's the best way to set up auto-scaling for my Kubernetes cluster on AWS? - """, - """ - I need to implement a zero-downtime deployment strategy for my microservices. - What approaches would you recommend? - """, - """ - How can I improve the security of my CI/CD pipeline and prevent supply chain attacks? - """, - """ - What monitoring metrics should I track to ensure the reliability of my distributed system? + What's the differnce between SQL and NoSQL databases, and when should I use each? """, ], fail_queries=[ """ - How do I design an effective user onboarding flow for my mobile app? + What's the best approach for setting up a CI/CD pipeline for our team? """, """ - What's the best algorithm for sentiment analysis on customer reviews? + Can you help me configure auto-scaling for our Kubernetes cluster? """, """ - Can you help me with color theory for my website redesign? + How should I structure our cloud infrastructure for better cost efficiency? """, """ - I need advice on optimizing my SQL queries for a reporting dashboard. + How do I cook a delicious lasagna for dinner? """, ], ) -# Security Specialist Persona -security_specialist = PersonaMatchTest( - persona_name="security specialist", +# DevOps/SRE Engineer Persona +devops_sre = PersonaMatchTest( + persona_name="devops/sre engineer", persona_desc=""" - Expert in cybersecurity, application security, and secure system design. - Specializes in identifying and mitigating security vulnerabilities and threats. - Performs security assessments, penetration testing, and code security reviews. - Implements security controls like authentication, authorization, and encryption. - Knowledgeable about common attack vectors such as injection attacks, XSS, CSRF, and SSRF. - Experienced with security frameworks and standards like OWASP Top 10, NIST, and ISO 27001. - Designs secure architectures and implements defense-in-depth strategies. - Conducts security incident response and forensic analysis. - Implements security monitoring, logging, and alerting systems. - Stays current with emerging security threats and mitigation techniques. + Expert in infrastructure automation, deployment pipelines, and operational reliability. + Specializes in building and maintaining scalable, resilient, and secure infrastructure. + Proficient with cloud platforms (AWS, Azure, GCP), containerization, and orchestration. + Experienced with infrastructure as code, configuration management, and automation tools. + Skilled in implementing CI/CD pipelines, monitoring systems, and observability solutions. + Focuses on reliability, performance, security, and operational efficiency. + Practices site reliability engineering principles and DevOps methodologies. + + Designs and implements cloud infrastructure using services like compute, storage, + networking, and databases. + Creates infrastructure as code using tools like Terraform, CloudFormation, or Pulumi. + Configures and manages container orchestration platforms like Kubernetes and ECS. + Implements CI/CD pipelines using tools like Jenkins, GitHub Actions, GitLab CI, or CircleCI. + + Sets up comprehensive monitoring, alerting, and observability solutions. + Implements logging aggregation, metrics collection, and distributed tracing. + Creates dashboards and visualizations for system performance and health. + Designs and implements disaster recovery and backup strategies. + + Automates routine operational tasks and infrastructure maintenance. + Conducts capacity planning, performance tuning, and cost optimization. + Implements security best practices, compliance controls, and access management. + Performs incident response, troubleshooting, and post-mortem analysis. + + Designs for high availability, fault tolerance, and graceful degradation. + Implements auto-scaling, load balancing, and traffic management solutions. + Creates runbooks, documentation, and operational procedures. + Conducts chaos engineering experiments to improve system resilience. """, pass_queries=[ """ - How can I protect my web application from SQL injection attacks? + How do I set up a Kubernetes cluster with proper high availability? """, """ - What security controls should I implement for storing sensitive user data? + What's the best approach for implementing a CI/CD pipeline for our microservices? """, """ - How do I conduct a thorough security assessment of our cloud infrastructure? + How can I automate our infrastructure provisioning using Terraform? """, """ - What's the best approach for implementing secure authentication in my API? + What monitoring metrics should I track to ensure the reliability of our system? """, ], fail_queries=[ """ - How do I optimize the loading speed of my website? + How do I implement a sorting algorithm in Python? """, """ - What's the best way to implement responsive design for mobile devices? + What's the best way to structure my React components for a single-page application? """, """ Can you help me design a database schema for my e-commerce application? """, """ - How should I structure my React components for better code organization? - """, - ], -) - -# Mobile Developer Persona -mobile_developer = PersonaMatchTest( - persona_name="mobile developer", - persona_desc=""" - Expert in building native and cross-platform mobile applications for iOS and Android. - Specializes in mobile UI development, responsive layouts, and platform-specific - design patterns. - Proficient in Swift and SwiftUI for iOS, Kotlin for Android, and React Native or - Flutter for cross-platform. - Implements mobile-specific features like push notifications, offline storage, and - location services. - Optimizes mobile applications for performance, battery efficiency, and limited - network connectivity. - Experienced with mobile app architecture patterns like MVVM, MVC, and Redux. - Integrates with device hardware features including camera, biometrics, sensors, - and Bluetooth. - Familiar with app store submission processes, app signing, and distribution workflows. - Implements secure data storage, authentication, and API communication on mobile devices. - Designs and develops responsive interfaces that work across different screen sizes - and orientations. - - Implements sophisticated offline-first data synchronization strategies - for mobile applications, - handling conflict resolution, data merging, and background syncing when connectivity - is restored. - Uses technologies like Realm, SQLite, Core Data, and Room Database to enable seamless - offline - experiences in React Native and native apps. - - Structures Swift code following the MVVM (Model-View-ViewModel) architectural pattern - to create - maintainable, testable iOS applications. Implements proper separation of concerns - with bindings - between views and view models using Combine, RxSwift, or SwiftUI's native state management. - - Specializes in deep linking implementation for both Android and iOS, enabling app-to-app - communication, marketing campaign tracking, and seamless user experiences when navigating - between web and mobile contexts. Configures Universal Links, App Links, and custom URL - schemes. - - Optimizes battery usage for location-based features by implementing intelligent location - tracking - strategies, including geofencing, significant location changes, deferred location updates, - and - region monitoring. Balances accuracy requirements with power consumption constraints. - - Develops efficient state management solutions for complex mobile applications using Redux, - MobX, Provider, or Riverpod for React Native apps, and native state management approaches - for iOS and Android. - - Creates responsive mobile interfaces that adapt to different device orientations, - screen sizes, - and pixel densities using constraint layouts, auto layout, size classes, and flexible - grid systems. - """, - pass_queries=[ - """ - What's the best approach for implementing offline-first data synchronization in my mobile - app? - """, - """ - How should I structure my Swift code to implement the MVVM pattern effectively? - """, - """ - What's the most efficient way to handle deep linking and app-to-app communication on - Android? + How do I create a responsive layout using CSS Grid and Flexbox? """, """ - How can I optimize battery usage when implementing background location tracking? - """, - ], - fail_queries=[ - """ - How do I design a database schema with proper normalization for my web application? - """, - """ - What's the best approach for implementing a distributed caching layer in my microservices? - """, - """ - Can you help me set up a data pipeline for processing large datasets with Apache Spark? - """, - """ - How should I configure my load balancer to distribute traffic across my web servers? - """, - ], -) - -# Database Administrator Persona -database_administrator = PersonaMatchTest( - persona_name="database administrator", - persona_desc=""" - Expert in designing, implementing, and managing database systems for optimal performance and - reliability. - Specializes in database architecture, schema design, and query optimization techniques. - Proficient with relational databases like PostgreSQL, MySQL, Oracle, and SQL Server. - Implements and manages database security, access controls, and data protection measures. - Designs high-availability solutions using replication, clustering, and failover mechanisms. - Develops and executes backup strategies, disaster recovery plans, and data retention - policies. - Monitors database performance, identifies bottlenecks, and implements optimization - solutions. - Creates and maintains indexes, partitioning schemes, and other performance-enhancing - structures. - Experienced with database migration, version control, and change management processes. - Implements data integrity constraints, stored procedures, triggers, and database automation. - - Optimizes complex JOIN query performance in PostgreSQL through advanced techniques including - query rewriting, proper indexing strategies, materialized views, and query plan analysis. - Uses EXPLAIN ANALYZE to identify bottlenecks in query execution plans and implements - appropriate optimizations for specific query patterns. - - Designs and implements high-availability MySQL configurations with automatic failover using - technologies like MySQL Group Replication, Galera Cluster, Percona XtraDB Cluster, or MySQL - InnoDB Cluster with MySQL Router. Configures synchronous and asynchronous replication - strategies - to balance consistency and performance requirements. - - Develops sophisticated indexing strategies for tables with frequent write operations and - complex - read queries, balancing write performance with read optimization. Implements partial - indexes, - covering indexes, and composite indexes based on query patterns and cardinality analysis. - - Specializes in large-scale database migrations between different database engines, - particularly - Oracle to PostgreSQL transitions. Uses tools like ora2pg, AWS DMS, and custom ETL processes - to - ensure data integrity, schema compatibility, and minimal downtime during migration. - - Implements table partitioning schemes based on data access patterns, including range - partitioning - for time-series data, list partitioning for categorical data, and hash partitioning for - evenly - distributed workloads. - - Configures and manages database connection pooling, query caching, and buffer management to - optimize resource utilization and throughput under varying workloads. - - Designs and implements database sharding strategies for horizontal scaling, including - consistent hashing algorithms, shard key selection, and cross-shard query optimization. - """, - pass_queries=[ - """ - How can I optimize the performance of complex JOIN queries in my PostgreSQL database? - """, - """ - What's the best approach for implementing a high-availability MySQL setup with automatic - failover? - """, - """ - How should I design my indexing strategy for a table with frequent writes and complex read - queries? - """, - """ - What's the most efficient way to migrate a large Oracle database to PostgreSQL with minimal - downtime? - """, - ], - fail_queries=[ - """ - How do I structure my React components to implement the Redux state management pattern? - """, - """ - What's the best approach for implementing responsive design with CSS Grid and Flexbox? - """, - """ - Can you help me set up a CI/CD pipeline for my containerized microservices? - """, - ], -) - -# Natural Language Processing Specialist Persona -nlp_specialist = PersonaMatchTest( - persona_name="nlp specialist", - persona_desc=""" - Expertise: Natural language processing, computational linguistics, and text analytics. - Proficient with NLP libraries and frameworks like NLTK, spaCy, Hugging Face Transformers, - and Gensim. - Experience with language models such as BERT, GPT, T5, and their applications. - Skilled in text preprocessing, tokenization, lemmatization, and feature extraction - techniques. - Knowledge of sentiment analysis, named entity recognition, topic modeling, and text - classification. - Familiar with word embeddings, contextual embeddings, and language representation methods. - Understanding of machine translation, question answering, and text summarization systems. - Background in information retrieval, semantic search, and conversational AI development. - """, - pass_queries=[ - """ - What approach should I take to fine-tune BERT for my custom text classification task? - """, - """ - How can I improve the accuracy of my named entity recognition system for medical texts? - """, - """ - What's the best way to implement semantic search using embeddings from language models? - """, - """ - I need to build a sentiment analysis system that can handle sarcasm and idioms. - Any suggestions? - """, - ], - fail_queries=[ - """ - How do I optimize my React components to reduce rendering time? - """, - """ - What's the best approach for implementing a CI/CD pipeline with Jenkins? - """, - """ - Can you help me design a responsive UI for my web application? - """, - """ - How should I structure my microservices architecture for scalability? + What's the most efficient algorithm for finding the shortest path in a graph? """, ], ) @@ -557,17 +305,12 @@ class PersonaMatchTest(BaseModel): "persona_match_test", [ simple_persona, - software_architect, - data_scientist, - ux_designer, - devops_engineer, - security_specialist, - mobile_developer, - database_administrator, - nlp_specialist, + architect, + coder, + devops_sre, ], ) -async def test_check_persona_match( +async def test_check_persona_pass_match( semantic_router_mocked_db: SemanticRouter, persona_match_test: PersonaMatchTest ): """Test checking persona match.""" @@ -582,9 +325,54 @@ async def test_check_persona_match( ) assert match is True + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "persona_match_test", + [ + simple_persona, + architect, + coder, + devops_sre, + ], +) +async def test_check_persona_fail_match( + semantic_router_mocked_db: SemanticRouter, persona_match_test: PersonaMatchTest +): + """Test checking persona match.""" + await semantic_router_mocked_db.add_persona( + persona_match_test.persona_name, persona_match_test.persona_desc + ) + # Check for the queries that should fail for query in persona_match_test.fail_queries: match = await semantic_router_mocked_db.check_persona_match( persona_match_test.persona_name, query ) assert match is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "personas", + [ + [ + coder, + devops_sre, + architect, + ] + ], +) +async def test_persona_diff_description( + semantic_router_mocked_db: SemanticRouter, + personas: List[PersonaMatchTest], +): + # First, add all existing personas + for persona in personas: + await semantic_router_mocked_db.add_persona(persona.persona_name, persona.persona_desc) + + last_added_persona = personas[-1] + with pytest.raises(PersonaSimilarDescriptionError): + await semantic_router_mocked_db.add_persona( + "repeated persona", last_added_persona.persona_desc + )