calculation with: http://justusrandolph.net/kappa/ Percent overall agreement = 61.04% Free-marginal kappa = 0.42 95% CI for free-marginal kappa [0.37, 0.46] Fixed-marginal kappa = 0.22 95% CI for fixed-marginal kappa [0.19, 0.24] ---- output of python script: 0.40 | 6.00 9.00 0.00 | air 1.00 | 15.00 0.00 0.00 | android 0.07 | 1.00 13.00 1.00 | anger 0.86 | 12.00 2.00 1.00 | apple 0.33 | 5.00 10.00 0.00 | bachelor 0.67 | 10.00 5.00 0.00 | ball 0.57 | 8.00 6.00 1.00 | bank 0.64 | 9.00 5.00 1.00 | bar 0.73 | 11.00 4.00 0.00 | bat 0.36 | 5.00 9.00 1.00 | bee 0.80 | 12.00 3.00 0.00 | beef 0.07 | 1.00 13.00 1.00 | beer 0.60 | 9.00 6.00 0.00 | bone 0.53 | 8.00 7.00 0.00 | bush 0.13 | 2.00 13.00 0.00 | canoe 0.47 | 7.00 8.00 0.00 | capital 0.07 | 1.00 13.00 1.00 | castle 0.53 | 8.00 7.00 0.00 | cat 0.36 | 5.00 9.00 1.00 | chest 0.57 | 8.00 6.00 1.00 | chicken 0.86 | 12.00 2.00 1.00 | clip 0.93 | 14.00 1.00 0.00 | cloud 0.33 | 5.00 10.00 0.00 | cocaine 0.40 | 6.00 9.00 0.00 | cow 0.80 | 12.00 3.00 0.00 | crane 0.00 | 0.00 15.00 0.00 | danger 0.87 | 13.00 2.00 0.00 | date 0.07 | 1.00 14.00 0.00 | day 0.60 | 9.00 6.00 0.00 | dog 0.67 | 10.00 5.00 0.00 | edge 0.60 | 9.00 6.00 0.00 | electricity 0.20 | 3.00 12.00 0.00 | elephant 0.73 | 11.00 4.00 0.00 | fan 0.27 | 4.00 11.00 0.00 | feeling 0.27 | 4.00 11.00 0.00 | fish 0.07 | 1.00 14.00 0.00 | forest 0.27 | 4.00 11.00 0.00 | hat 0.53 | 8.00 7.00 0.00 | head 0.53 | 8.00 7.00 0.00 | honey 0.13 | 2.00 13.00 0.00 | house 0.13 | 2.00 13.00 0.00 | knight 0.07 | 1.00 14.00 0.00 | lake 0.40 | 6.00 9.00 0.00 | lamp 0.40 | 6.00 9.00 0.00 | language 0.27 | 4.00 11.00 0.00 | leaf 0.20 | 3.00 12.00 0.00 | letter 0.20 | 3.00 12.00 0.00 | lightning 0.27 | 4.00 11.00 0.00 | love 0.47 | 7.00 8.00 0.00 | mail 0.33 | 5.00 10.00 0.00 | meat 0.20 | 3.00 12.00 0.00 | meeting 0.73 | 11.00 4.00 0.00 | metal 0.93 | 13.00 1.00 1.00 | monitor 0.47 | 7.00 8.00 0.00 | muscle 0.40 | 6.00 9.00 0.00 | nail 0.13 | 2.00 13.00 0.00 | night 0.33 | 5.00 10.00 0.00 | novel 0.27 | 4.00 11.00 0.00 | number 0.07 | 1.00 14.00 0.00 | ocean 0.47 | 7.00 8.00 0.00 | paper 0.60 | 9.00 6.00 0.00 | pen 0.60 | 9.00 6.00 0.00 | pig 0.40 | 6.00 9.00 0.00 | plant 0.27 | 4.00 11.00 0.00 | pond 0.47 | 7.00 8.00 0.00 | pork 0.93 | 14.00 1.00 0.00 | pot 0.07 | 1.00 14.00 0.00 | poultry 0.50 | 7.00 7.00 1.00 | power 0.20 | 3.00 12.00 0.00 | priest 0.87 | 13.00 2.00 0.00 | profile 0.47 | 7.00 8.00 0.00 | right 0.07 | 1.00 14.00 0.00 | river 0.60 | 9.00 6.00 0.00 | rock 0.60 | 9.00 6.00 0.00 | sandbox 0.13 | 2.00 13.00 0.00 | sea 0.13 | 2.00 13.00 0.00 | skull 0.27 | 4.00 11.00 0.00 | slave 0.07 | 1.00 14.00 0.00 | smell 0.40 | 6.00 9.00 0.00 | snow 0.53 | 8.00 7.00 0.00 | state 0.47 | 7.00 8.00 0.00 | stone 0.20 | 3.00 12.00 0.00 | stranger 0.40 | 6.00 9.00 0.00 | subject 0.07 | 1.00 14.00 0.00 | sword 0.53 | 8.00 7.00 0.00 | table 1.00 | 15.00 0.00 0.00 | tablet 0.67 | 10.00 5.00 0.00 | tank 0.13 | 2.00 13.00 0.00 | taste 0.87 | 13.00 2.00 0.00 | text 0.27 | 4.00 11.00 0.00 | thumb 0.13 | 2.00 13.00 0.00 | thunder 0.47 | 7.00 8.00 0.00 | tie 0.40 | 6.00 9.00 0.00 | toast 0.73 | 11.00 4.00 0.00 | train 0.33 | 5.00 10.00 0.00 | tree 1.00 | 15.00 0.00 0.00 | troll 0.43 | 6.00 8.00 1.00 | volley 0.53 | 8.00 7.00 0.00 | wagon 0.20 | 3.00 12.00 0.00 | water 0.33 | 5.00 10.00 0.00 | wheel 0.07 | 1.00 14.00 0.00 | wine 0.20 | 3.00 12.00 0.00 | wood 0.07 | 1.00 13.00 1.00 | year 0.27 | 4.00 11.00 0.00 | axes 0.57 | 8.00 6.00 1.00 | bass 0.29 | 4.00 10.00 1.00 | motion 0.27 | 4.00 11.00 0.00 | palm 0.20 | 3.00 12.00 0.00 | poach 0.36 | 5.00 9.00 1.00 | sake 0.67 | 10.00 5.00 0.00 | space 0.73 | 11.00 4.00 0.00 | depression 0.87 | 13.00 2.00 0.00 | plane clear 0: 12 | clear 1: 22 | majority 0: 42 | majority 1: 70 responses = { 'Gained an additional sense since 1800': 0, 'No additional sense since 1800': 1, 'NO_ANSWER': 2, # "I don't know this word": 3 } {'air': 0, 'android': 1, 'anger': 2, 'apple': 3, 'bachelor': 4, 'ball': 5, 'bank': 6, 'bar': 7, 'bat': 8, 'bee': 9, 'beef': 10, 'beer': 11, 'bone': 12, 'bush': 13, 'canoe': 14, 'capital': 15, 'castle': 16, 'cat': 17, 'chest': 18, 'chicken': 19, 'clip': 20, 'cloud': 21, 'cocaine': 22, 'cow': 23, 'crane': 24, 'danger': 25, 'date': 26, 'day': 27, 'dog': 28, 'edge': 29, 'electricity': 30, 'elephant': 31, 'fan': 32, 'feeling': 33, 'fish': 34, 'forest': 35, 'hat': 36, 'head': 37, 'honey': 38, 'house': 39, 'knight': 40, 'lake': 41, 'lamp': 42, 'language': 43, 'leaf': 44, 'letter': 45, 'lightning': 46, 'love': 47, 'mail': 48, 'meat': 49, 'meeting': 50, 'metal': 51, 'monitor': 52, 'muscle': 53, 'nail': 54, 'night': 55, 'novel': 56, 'number': 57, 'ocean': 58, 'paper': 59, 'pen': 60, 'pig': 61, 'plant': 62, 'pond': 63, 'pork': 64, 'pot': 65, 'poultry': 66, 'power': 67, 'priest': 68, 'profile': 69, 'right': 70, 'river': 71, 'rock': 72, 'sandbox': 73, 'sea': 74, 'skull': 75, 'slave': 76, 'smell': 77, 'snow': 78, 'state': 79, 'stone': 80, 'stranger': 81, 'subject': 82, 'sword': 83, 'table': 84, 'tablet': 85, 'tank': 86, 'taste': 87, 'text': 88, 'thumb': 89, 'thunder': 90, 'tie': 91, 'toast': 92, 'train': 93, 'tree': 94, 'troll': 95, 'volley': 96, 'wagon': 97, 'water': 98, 'wheel': 99, 'wine': 100, 'wood': 101, 'year': 102, 'axes': 103, 'bass': 104, 'motion': 105, 'palm': 106, 'poach': 107, 'sake': 108, 'space': 109, 'depression': 110, 'plane': 111} [0.42 0.57 0.01] 1.0000000000000002 [0.49 1. 0.74 0.64 0.52 0.52 0.41 0.44 0.58 0.44 0.66 0.74 0.49 0.47 0.75 0.47 0.74 0.47 0.44 0.41 0.64 0.87 0.52 0.49 0.66 1. 0.75 0.87 0.49 0.52 0.49 0.66 0.58 0.58 0.58 0.87 0.58 0.47 0.47 0.75 0.75 0.87 0.49 0.49 0.58 0.66 0.66 0.58 0.47 0.52 0.66 0.58 0.74 0.47 0.49 0.75 0.52 0.58 0.87 0.47 0.49 0.49 0.49 0.58 0.47 0.87 0.87 0.4 0.66 0.75 0.47 0.87 0.49 0.49 0.75 0.75 0.58 0.87 0.49 0.47 0.47 0.66 0.49 0.87 0.47 1. 0.52 0.75 0.75 0.58 0.75 0.47 0.49 0.58 0.52 1. 0.41 0.47 0.66 0.52 0.87 0.66 0.74 0.58 0.41 0.49 0.58 0.66 0.44 0.52 0.58 0.75] 0.6103741496598639 0.5013740079365081 kappa: 0.2186010024713602