{"id":"9cd7c0ef-4b55-4af5-b3d8-51a434f9a8cf","company":{"id":"6e9b4866-fa55-42ed-addf-8448051a14fa","company_stack":[],"company_gallery":[],"company_stats":[],"company_about_section":{"name":"FM India","video_link":null,"description":"FM is a 190-year-old, Fortune 500 commercial property insurance company of 6,000+ employees with a unique focus on science and risk engineering. Serving over a quarter of the Fortune 500 and major corporations globally, they deliver data-driven strategies that enhance resilience, ensure business continuity, and empower organizations to thrive. \r\nFM India located in Bengaluru is a strategic location for driving FM's global operational efficiency that allows them to leverage the country’s talented workforce and advance their capabilities to serve their clients better.","primary_color":null,"secondary_color":null},"company_benefit_section":[],"company_testimonial_section":[],"company_culture_section":[],"onboarding_process":null,"ticker_logos":[],"gtn_companies":[],"role_category":[{"name":"BE Technology Workforce Development"},{"name":"Business Enablement"},{"name":"Business Operations"},{"name":"Human Resources"},{"name":"Natural Hazard & Aggregation"},{"name":"Technology"}],"cta_color":"#333333","name":"FM India","short_name":"FM India","slug":"fmindia","description":"FM is a 190-year-old, Fortune 500 commercial property insurance company of 6,000+ employees with a unique focus on science and risk engineering. Serving over a quarter of the Fortune 500 and major corporations globally, they deliver data-driven strategies that enhance resilience, ensure business continuity, and empower organizations to thrive. \r\nFM India located in Bengaluru is a strategic location for driving FM's global operational efficiency that allows them to leverage the country’s talented workforce and advance their capabilities to serve their clients better.","short_description":"<p>We are a highly successful 190-year-old, Fortune 500 commercial property insurance company of 6,000+ employees with a unique focus on science and risk engineering. Businesses worldwide trust our expertise to protect their assets, relying on our comprehensive risk assessments and robust, engineering-based insurance solutions to safeguard against fire, natural disasters, and other perils. Serving over a quarter of the Fortune 500 and major corporations globally, we deliver data-driven strategies that enhance resilience, ensure business continuity, and empower organizations to thrive.</p>","about_us":"We are a highly successful 190-year-old, Fortune 500 commercial property insurance company of 6,000+ employees with a unique focus on science and risk engineering. Businesses worldwide trust our expertise to protect their assets, relying on our comprehensive risk assessments and robust, engineering-based insurance solutions to safeguard against fire, natural disasters, and other perils. Serving over a quarter of the Fortune 500 and major corporations globally, we deliver data-driven strategies that enhance resilience, ensure business continuity, and empower organizations to thrive.","video_link":null,"logo_url":"https://talent500-public-prod-cb.s3.ap-south-1.amazonaws.com/company_logos/image-11703_1_d1783fec41e6489cac47815f7e71e96b.png","job_logo":null,"job_logo_url":"","show_powered_by_talent500_logo":false,"website":null,"redirect_logo_url":"https://talent500.com/fmglobal/home/","linkedIn":null,"facebook":null,"twitter":null,"instagram":null,"hero_type":"1","nav_bar_color":null,"nav_items_color":"#FFFFFF","gallery_heading":"Gallery","company_image_url":"","meta_title":null,"meta_description":"For nearly two centuries, businesses worldwide have trusted us, a global leader in commercial property insurance, to protect their assets. We provide comprehensive risk assessments, robust insurance plans, and innovative, engineering-based solutions protecting against fire, natural disasters, and other perils. Serving over 25% of the Fortune 500 companies and large corporations worldwide, we deliver data-driven risk management solutions that enhance business resilience and continuity and empower businesses to thrive.\r\n\r\nFM India, located in Bengaluru, is our second HQ and a key hub for driving our global operational efficiency. Our presence in India allows us to leverage the country’s talented workforce and advance our capabilities to serve our clients better. We have diverse corporate functions that emphasize research, advanced technologies like AI and analytics, risk engineering, research, finance, marketing, HR, etc.","hero_h1":null,"hero_h2":null,"primary_color":"#0D102B","secondary_color":"#333333","cta_text":null,"jobs_subtitle":null,"mission":null,"stack_section_title":null,"stack_section_description":"","powered_by_logo_url":"https://d36dj8u11ystvj.cloudfront.net/Talent500_PoweredBy+(1).png","cta_bg_color":"#FFFFFF","cta_text_color":"#FFA000","cta_border_color":"#FFFFFF","h1_font_size":null,"h2_font_size":null,"h3_font_size":null,"h4_font_size":null,"body_size_1":null,"body_size_2":null,"h1_font_w":null,"h2_font_w":null,"h3_font_w":null,"h4_font_w":null,"body_w_1":null,"body_w_2":null,"public_name":null,"use_public_name":false,"unique_id":"cf0459b0-1760-4f5b-96d6-20436cb13179","head_favicon_url":"","head_title":null,"privacy_policy_name":"Privacy Policy","privacy_policy_url":null,"mandate_privacy_policy":null,"ticker_title":null,"publish":true,"company_font_family":"5","use_company_font_on_jobs_page":true,"gallery_section_name":null,"about_section_name":"FM India","testimonial_section_name":null,"benefits_section_name":null,"culture_section_name":null,"jobs_section_name":null,"is_job_displayable":true,"is_onboarding_skip":false,"company_specific_workflow":0,"third_party_company":false,"is_vendor_third_party_company":false,"priority_order":0,"order_of_about_role":1,"order_of_about_company":2,"whatsapp_company_logo":"uploads/company_images/FMIndia.png","whatsapp_company_logo_url":"https://talent500-public-prod-cb.s3.ap-south-1.amazonaws.com/talent500-companies/whatsapp/logos/FMIndia_e539ac4b146a463397d66aa151cf27ee.png","whatsapp_job_bg_color":"#040812","whatsapp_job_text_color":"#FFFFFF","engagement_type":null,"font_family":5},"location":"Bengaluru","primary_skills":["Gen AI or Generative AI","Azure","Python","AI or ML"],"secondary_skills":["LLM or Open AI or Cloud Models or GPT","RAG"],"experience_range":"1 - 3 years","job_qualifications":[""],"screening_questions":[{"id":"e82da328-60d5-4c43-9057-726db22a5da1","question_number":1,"description":"How many years of experience do you have in AI/ML?","question_type":"MCSA","question_options":[{"option":"Less than 6 months","option_id":1},{"option":"6-12 months","option_id":2},{"option":"More than 1 year","option_id":3},{"option":"More than 2 years","option_id":4},{"option":"No experience, but have strong understanding & knowledge","option_id":5},{"option":"No experience, but, I'm a quick learner","option_id":6}]},{"id":"df122974-7667-48b1-bc3f-1cec19c1aaeb","question_number":2,"description":"How many years of experience do you have in Gen AI?","question_type":"MCSA","question_options":[{"option":"Less than 6 months","option_id":1},{"option":"6-12 months","option_id":2},{"option":"More than 1 year","option_id":3},{"option":"More than 2 years","option_id":4},{"option":"No experience, but have strong understanding & knowledge","option_id":5},{"option":"No experience, but, I'm a quick learner","option_id":6}]},{"id":"335fabec-dfd0-4583-aae3-e2da796fb0b0","question_number":3,"description":"How many years of experience do you have in Azure?","question_type":"MCSA","question_options":[{"option":"Less than 6 months","option_id":1},{"option":"6-12 months","option_id":2},{"option":"More than 1 year","option_id":3},{"option":"More than 2 years","option_id":4},{"option":"No experience, but have strong understanding & knowledge","option_id":5},{"option":"No experience, but, I'm a quick learner","option_id":6}]},{"id":"bef0c2f6-abc2-42c4-b24f-84a96dbbc577","question_number":4,"description":"How many years of experience do you have in Python?","question_type":"MCSA","question_options":[{"option":"Less than 6 months","option_id":1},{"option":"6-12 months","option_id":2},{"option":"More than 1 year","option_id":3},{"option":"More than 2 years","option_id":4},{"option":"No experience, but have strong understanding & knowledge","option_id":5},{"option":"No experience, but, I'm a quick learner","option_id":6}]},{"id":"36f35b28-bb56-40d2-b7a3-3a8af6af73a8","question_number":5,"description":"If selected, how soon can you join us?","question_type":"MCSA","question_options":[{"option":"Within 15 days","option_id":1},{"option":"Within 15-30 days","option_id":2},{"option":"Within 30-45 days","option_id":3},{"option":"Within 45-60 days","option_id":4},{"option":"60+ days","option_id":5}]},{"id":"adfc0829-67f2-4304-b924-b4789cb16b95","question_number":6,"description":"If selected, would you be willing to relocate to Bangalore?","question_type":"MCSA","question_options":[{"option":"Yes, willing to relocate","option_id":1},{"option":"No, not willing to relocate","option_id":2},{"option":"I am based in Bangalore","option_id":3}]},{"id":"71c98e33-8785-4c41-affa-0febae290361","question_number":7,"description":"What is your overall years of experience?","question_type":"MCSA","question_options":[{"option":"0-1 year","option_id":1},{"option":"1-3 years","option_id":2},{"option":"3-5 years","option_id":3},{"option":"5+ years","option_id":4}]}],"title":"Senior GenAI Engineer (AI Evaluation Engineer)","description":"<p><strong>GenAI Engineer (AI Evaluation Engineer)</strong></p><p><strong>Reports To Manager of GenAI Engineering</strong></p><p><strong>Job Summary:</strong></p><p>The <strong>Gen AI Evaluation Engineer</strong> leads the design, implementation, and operation of <strong>enterprise-grade evaluation, quality, and governance frameworks</strong> for Generative AI systems in a highly regulated, responsible AI environment. This role ensures the <strong>quality, reliability, safety, and performance</strong> of LLMs, vision models, RAG pipelines, and agentic workflows deployed in production.</p><p>Building on strong GenAI engineering foundations, this position focuses on <strong>AI-specific testing, experimentation, automation, and continuous evaluation pipelines</strong>, integrating quality gates into CI/CD workflows and aligning GenAI solutions with enterprise architecture, compliance, and risk standards. The Gen AI Evaluation Engineer partners closely with product, data science, ML engineering, and platform teams to drive <strong>trustworthy, scalable, and production-ready AI systems</strong>.</p><p><strong>Essential Functions &amp; Responsibilities:</strong></p><p><strong>GenAI Application Design &amp; Implementation:</strong></p><ul><li>Design and implement <strong>comprehensive AI evaluation and experimentation frameworks</strong> for LLMs, vision models, RAG pipelines, and agentic workflows.</li><li>Build <strong>automated evaluation systems</strong> to assess model outputs for accuracy, relevance, bias, hallucinations, safety, and regression stability.</li><li>Develop <strong>quality benchmarks and continuous testing pipelines</strong> covering content quality, safety, alignment, and enterprise compliance.</li><li>Establish <strong>AI-specific quality gates</strong> and acceptance criteria integrated into Agile sprints and CI/CD pipelines.</li><li>Design, develop, and evaluate <strong>data pipelines and RAG workflows</strong> using Promptflow, Azure AI Search, ADF Pipelines, Databricks, Spark, and Vector Databases.</li><li>Validate prompt engineering strategies, prompt consistency, and inference pipelines using GenAI-specific testing tools.</li><li>Perform prompt-based scenario testing, hallucination detection, and regression validation across model versions.</li></ul><p><strong>System Support &amp; Operational Excellence:</strong></p><ul><li>Develop and maintain <strong>monitoring capabilities</strong> for model drift detection, data quality validation, inference latency, and system reliability.</li><li>Monitor production GenAI systems using <strong>Azure Application Insights, Dynatrace</strong>, and AI evaluation platforms.</li><li>Apply <strong>AI risk-focused testing</strong>, including bias, fairness, safety, adversarial testing, and red-teaming methodologies.</li><li>Ensure optimized utilization of infrastructure and compute across diverse AI workloads.</li></ul><p><strong>Automation &amp; Test Engineering:</strong></p><ul><li>Build and maintain <strong>test automation frameworks</strong> (primarily in Python) for AI/ML evaluation and end-to-end workflow validation.</li><li>Write automation scripts to <strong>simulate user behavior and backend interactions</strong>.</li><li>Perform exploratory testing, regression testing, and end-to-end system validation for AI-enabled applications.</li><li>Track and manage defects using AI evaluation platforms and Agile tools; document test plans, execution results, and evaluation metrics.</li></ul><p><strong>Mentorship &amp; Team Leadership:</strong></p><ul><li>Lead design and evaluation reviews; mentor junior engineers and evaluation specialists.</li><li>Collaborate with product managers to convert business and regulatory requirements into <strong>test cases, benchmarks, and test data</strong>.</li><li>Drive innovation by adopting emerging GenAI evaluation techniques and aligning AI initiatives with enterprise goals.</li><li>Foster a culture of <strong>technical excellence, responsible AI, and continuous improvement</strong>.</li></ul><p><strong>Essential Skills:</strong></p><ul><li>Design and implementation of <strong>AI evaluation frameworks</strong> for LLMs, RAG, vision models, and agentic systems</li><li>Generative AI engineering: prompt engineering, model integration, inference pipelines</li><li>Data pipelines and RAG workflows (Promptflow, Azure AI Search, ADF Pipelines, Vector DBs)</li><li>Automated testing and CI/CD integration for AI systems</li><li>Monitoring and observability (Azure Application Insights, Dynatrace)</li><li>AI risk, safety, bias, fairness, hallucination detection, and adversarial testing</li><li>Large-scale experimentation and performance optimization</li><li>Strong mentorship, technical leadership, and cross-team collaboration</li></ul><p><strong>Must Have Skills:</strong></p><ul><li>Generative AI&nbsp;</li><li>Azure (AI, Data, and Monitoring services)</li><li>AI/ML Platform tools (Databricks, Azure AI/ML, GCP Vertex, AWS Bedrock)</li><li>Python (test automation, evaluation frameworks)</li><li>Software Engineering &amp; Test Engineering fundamentals</li></ul><p><strong>Basic Qualifications:</strong></p><ul><li><strong>1-3 years of experience</strong> in evaluation or test engineering with a strong focus on AI/ML and Generative AI systems.</li><li>Hands-on experience evaluating <strong>LLMs, RAG pipelines, and AI-powered applications</strong> in production environments.</li><li>Strong understanding of AI/ML evaluation concepts including <strong>accuracy, relevance, regression, latency, and stability</strong>.</li><li>Bachelor’s Degree in Computer Science, Data Science, Artificial Intelligence, or equivalent practical experience.</li></ul><p><strong>Preferred Qualifications:</strong></p><ul><li>Knowledge of AI risks including <strong>bias, fairness, safety, hallucinations, and adversarial attacks</strong>.</li><li>Experience in highly regulated or governed AI environments.</li><li>Strong collaboration skills with data scientists, ML engineers, platform teams, and product managers.</li><li>Passion for AI quality, responsible AI practices, and staying current with <strong>emerging GenAI evaluation methodologies</strong>.</li></ul>","employment_type":"Full-Time","industry":"","job_template":false,"country":{"name":"India","country_code":"IN","region":"Asia"},"is_job_displayable":true,"job_url":"https://talent500.com/jobs/fmindia/senior-genai-engineer-ai-evaluation-engineer-bengaluru-T500-26087","created_at":"2026-05-21T11:38:34.434480+05:30","updated_at":"2026-06-25T14:24:07.294758+05:30","title_alias_1":"Senior GenAI Engineer (AI Evaluation Engineer)","slug":"senior-genai-engineer-ai-evaluation-engineer-bengaluru-T500-26087","summary":null,"typical_workday":null,"what_you_offer":null,"what_you_need_to_succeed":null,"responsibilities":null,"type":null,"min_experience_years":1,"max_experience_years":3,"ctc_unit":"LPA","is_featured_job":false,"is_active":true,"job_code":"T500-26087","external_id":38072,"external_job_code":"FMJP015","is_anonymized":false,"mercurial_skills":null,"open_date":"2026-05-21T11:31:21.615606+05:30","published_at":"2026-05-21T11:38:33.710197+05:30","anonymization_needed":null,"anonymized_description":null,"xml_feed_anonymization":null,"unregistered_user_anonymization":null,"registered_user_anonymization":null,"pre_application_email_anonymization":null,"mettl_account":null,"job_table_sub_head":null,"job_table_short_desc":null,"is_remote":null,"job_video":null,"job_video_title":null,"job_video_description":null,"job_image_url":"https://wattt.s3.ap-south-1.amazonaws.com/PRODUCTION/T500-26087.png","role_summary":"<p><strong>About FM:</strong><br>FM is a 190-year-old, Fortune 500 commercial property insurance company of 6,000+ employees with a unique focus on science and risk engineering. Serving over a quarter of the Fortune 500 and major corporations globally, they deliver data-driven strategies that enhance resilience, ensure business continuity, and empower organizations to thrive.&nbsp;<br>FM India located in Bengaluru is a strategic location for driving FM's global operational efficiency that allows them to leverage the country’s talented workforce and advance their capabilities to serve their clients better.</p>","syndication_control":true,"syndication_status":"in_progress","syndication_job_title_alias":"[]","sourcing_requirements_updated_at":null,"t500_sourcing_control":true,"status":"open","job_redirection_url":null,"is_external_job":false,"is_leadership_job":false,"leadership_confidentiality_level":null,"pricing_type":null,"is_auto_apply_eligible":false,"external_apply_url":null,"ats_provider":null,"apply_timeout_seconds":null,"recruiter":"1154daea-f24c-4f7f-aad6-494543ee6112","category":47,"job_sub_category":54,"role":null,"job_function":794,"job_sub_function":1372,"naukri_industry":null,"naukri_functional_area":null,"naukri_role":null,"role_category":581,"linkedin_industry":null,"syndication_partners":["155c6cf4-8e99-4b9d-a962-30efd35c2aeb","56f3dcdd-b898-4806-ae12-c2578268f37c","5dc79af2-bfa4-4186-b589-219f71a4d5cf","9997eeaf-0206-4bb3-a458-e3823ad26e2f","a8bf9ce8-9d0e-401f-bb42-a409477787a6","aa8de7a4-71c0-4241-8523-33f47d7da6c6","dd8c2a4a-08ef-4849-8fd0-c5017b662841","f2b2a145-60a0-4c8e-86a2-dd03b3ee374a"],"candidate_job_status":null,"candidate_job_id":null,"mettl_assessment_to_be_taken":false,"company_font_data":{"font-family":"Montserrat","url":"https://fonts.googleapis.com/css2?family=Montserrat:ital,wght@0,100;0,200;0,300;0,400;0,500;0,600;0,700;0,800;0,900;1,100;1,200;1,300;1,400;1,500;1,600;1,700;1,800;1,900&display=swap"}}