Coverage for skema/program_analysis/tests/test_model_coverage.py: 100%

1import json 

2import pytest 

3import os 

4 

5# Dictionary that maps 

6# "model" : [supported lines, total lines] 

7# Last updated: January 26th, 2024 

8# (Updated Bucky model to a slightly smaller count, due to some unknown issue) 

9# Previous update: November 29th, 2023 

10# (Updated TIE-GCM to a smaller line count, to address potential timeout issues on the GitHub CI) 

11ALL_MODELS = { 

12 "CHIME-penn-full": [1080, 1080], 

13 "CHIME-SIR": [633, 633], 

14 "CHIME-SVIIvR": [539, 539], 

15 "ABM-COmplexVID-19": [1133, 1133], 

16 "ABM-COVID-ABS": [1094, 2729], 

17 "MechBayes": [0, 0], 

18 "SIDARTHE": [193, 193], 

19 "Simple-SIR": [53, 53], 

20 "Climlab-v1": [4306, 4306], 

21 "Generated-Halfar": [128, 128], 

22 "SV2AIR3-Waterloo-MATLAB": [0, 1020], 

23 "Bucky": [7537, 7537], 

24 "ABM-REINA": [2622, 7078], 

25 "Cornell-COVID19-sim-Frazier": [7250, 8725], 

26 "ABM-Covasim": [14734, 31042], 

27 "TIE-GCM": [6336, 209076] 

28} 

29 

30# REPORTS_FILE_PATH = "/Users/ferra/Desktop/Work_Repos/skema/reports" 

31REPORTS_FILE_PATH = os.getenv("GITHUB_WORKSPACE") 

32LINE_COVERAGE_FILE_NAME = "line_coverage.json" 

33TEST_COVERAGE_LOCATION = os.path.join(REPORTS_FILE_PATH, "docs", "coverage", "code2fn_coverage", LINE_COVERAGE_FILE_NAME) 

34 

35 

36def load_line_coverage_information(): 

37 """  

38 Loads the most recently generated test coverage  

39 information 

40 """ 

41 return json.load(open(TEST_COVERAGE_LOCATION)) 

42 

43def test_all_models(): 

44 """ 

45 Tests the coverage of every model we have support for 

46 Also checks to make sure that we've covered every model that we currently support 

47 """ 

48 coverage_models = load_line_coverage_information() 

49 models_visited = 0 

50 for model in ALL_MODELS.keys(): 

51 baseline_supported_lines, baseline_total_lines = ALL_MODELS[model] 

52 current_supported_lines, current_total_lines = coverage_models[model] 

53 assert current_supported_lines >= baseline_supported_lines, f"model {model} supported line count has decreased" 

54 assert current_total_lines >= baseline_total_lines, f"model {model} total line count has decreased" 

55 models_visited += 1 

56 

57 # In case the coverage report generation doesn't give us all the models back, we use this assertion to check that 

58 assert models_visited == len(ALL_MODELS.keys()), f"test_all_models didn't test all {len(ALL_MODELS.keys())} models, only tested {models_visited} models"