Measurement Assessment

I now have all the data collected (unless I have to throw some out and replace it), so let’s look at some descriptive statistics!

First, import packages.

# import packages
library(broom)
library(corrr)
library(tidyverse)

library(flextable)

library(psych)
library(lavaan)
library(semTools)

Import my R objects from previous documents.

formatAsTable <- readRDS("format.rds")
adhd.data <- readRDS(file.path("..", "data", "quality-data.rds"))

Inter-Item Reliabilities

What are the reliabilities of the measures? Using the lavaan package, I will do a confirmatory factor analysis of the twelve items. I’m planning to use McDonald’s \(\omega\) in addition to Cronbach’s \(\alpha\) because it performs better and is preferable especially where there is skew.

cfa.model <- 'aff =~ aff1 + aff2 + aff3 + aff4
              cog =~ cog1 + cog2 + cog3 + cog4
              lik =~ lik1 + lik2 + lik3 + lik4'

cfa.fit <- cfa(cfa.model, adhd.data, effect.coding = T)

cfa.fit %>%
  reliability %>%
  as_tibble(rownames = "stat") %>%
  formatAsTable

stat	aff	cog	lik
alpha	0.91	0.44	-0.49
omega	0.88	0.55	0.07
omega2	0.88	0.55	0.07
omega3	0.88	0.60	0.06
avevar	0.71	0.41	0.50

The reliabilities for cog and lik are low because some of their items are reverse-coded. I will try again.

adhd.data <- adhd.data %>%
  mutate(across(c(cog4, lik3, lik4),
                fct_rev,
                .names = "{.col}.r"),
         lik4.r = fct_collapse(lik4.r,
                               agree = c("Strongly agree",
                                         "Somewhat agree")))

cfa.model <- 'aff =~ aff1 + aff2 + aff3 + aff4
              cog =~ cog1 + cog2 + cog3 + cog4.r
              lik =~ lik1 + lik2 + lik3.r + lik4.r'

cfa.fit <- cfa(cfa.model, adhd.data, effect.coding = T)

cfa.fit %>%
  reliability %>%
  as_tibble(rownames = "stat") %>%
  formatAsTable

stat	aff	cog	lik
alpha	0.91	0.66	0.80
omega	0.88	0.61	0.72
omega2	0.88	0.61	0.72
omega3	0.87	0.56	0.70
avevar	0.71	0.40	0.51

Ouch, cog was not a terribly reliable measure.

Exploratory Factor Analysis

adhd.data <- adhd.data %>%
  mutate(across(where(is.factor),
                ~ as.integer(.),
                .names = "{.col}.int"))

# Exploratory factor analysis with 3 factors
adhd.data %>%
  select(matches("\\w{3}\\d.int$")) %>%
  factanal(3) %>%
  tidy %>%
  formatAsTable

variable	uniqueness	fl1	fl2	fl3
aff1.int	0.39	0.77	0.07	-0.09
aff2.int	0.40	0.76	0.16	0.04
aff3.int	0.27	0.84	0.12	0.09
aff4.int	0.36	0.78	0.16	0.07
cog1.int	0.42	0.55	0.37	0.37
cog2.int	0.12	0.31	0.12	0.87
cog3.int	0.75	0.41	0.05	0.28
cog4.int	0.85	0.10	-0.13	-0.36
lik1.int	0.41	0.62	0.45	0.08
lik2.int	0.58	0.50	0.40	0.09
lik3.int	0.60	-0.05	-0.58	-0.25
lik4.int	0.40	-0.24	-0.73	-0.09

And let’s take another look, this time without cog4.

adhd.data %>%
  select(matches("\\w{3}\\d.int$") & -cog4.int) %>%
  factanal(3) %>%
  tidy %>%
  formatAsTable

variable	uniqueness	fl1	fl2	fl3
aff1.int	0.39	0.77	0.07	0.04
aff2.int	0.40	0.74	0.17	0.14
aff3.int	0.27	0.82	0.13	0.21
aff4.int	0.36	0.76	0.17	0.17
cog1.int	0.43	0.49	0.42	0.40
cog2.int	0.03	0.15	0.22	0.95
cog3.int	0.76	0.36	0.09	0.32
lik1.int	0.41	0.60	0.46	0.13
lik2.int	0.58	0.48	0.41	0.13
lik3.int	0.62	-0.02	-0.59	-0.18
lik4.int	0.39	-0.22	-0.75	-0.05

Confirmatory Factor Analysis

Anyway, as long as I have the CFA model, I might as well look at the loadings and fit statistics.

# loadings
cfa.fit %>%
  parameterEstimates %>%
  filter(op == "=~") %>%
  formatAsTable

lhs	op	rhs	est	se	z	pvalue	ci.lower	ci.upper
aff	=~	aff1	0.94	0.02	47.15	0.00	0.90	0.98
aff	=~	aff2	0.99	0.01	65.95	0.00	0.96	1.02
aff	=~	aff3	1.06	0.01	71.92	0.00	1.03	1.09
aff	=~	aff4	1.01	0.01	71.29	0.00	0.99	1.04
cog	=~	cog1	1.67	0.06	26.25	0.00	1.55	1.80
cog	=~	cog2	1.16	0.05	23.05	0.00	1.06	1.25
cog	=~	cog3	0.98	0.06	16.39	0.00	0.87	1.10
cog	=~	cog4.r	0.19	0.08	2.28	0.02	0.03	0.35
lik	=~	lik1	1.29	0.04	36.86	0.00	1.22	1.36
lik	=~	lik2	1.08	0.03	35.47	0.00	1.02	1.14
lik	=~	lik3.r	0.65	0.04	15.50	0.00	0.57	0.74
lik	=~	lik4.r	0.97	0.04	27.67	0.00	0.90	1.04

# fit statistics
m <- c("chisq", "df", "pvalue", "rmsea", "tli", "cfi")
cfa.fit %>%
  fitMeasures(fit.measures = m) %>%
  round(3) %>%
  as_tibble(rownames = 'stat') %>%
  formatAsTable

stat	value
chisq	290.53
df	51.00
pvalue	0.00
rmsea	0.10
tli	0.98
cfi	0.98

Interesting. It does look like cog4 deviated from the other cog items. Still loaded significantly, though.

I’m curious how this would all look if we dropped the potentially problematic observations I previously identified.

adhd.data.hq <- adhd.data %>%
  # exclude low-effort participants
  filter(!if_any(c(random_clicker, nonpart, ftl)))

cfa.fit.hq <- cfa.model %>%
  cfa(adhd.data.hq, effect.coding = T)

cfa.fit.hq %>%
  reliability %>%
  as_tibble(rownames = "stat") %>%
  formatAsTable

stat	aff	cog	lik
alpha	0.91	0.67	0.80
omega	0.88	0.61	0.73
omega2	0.88	0.61	0.73
omega3	0.88	0.57	0.71
avevar	0.71	0.41	0.51

cfa.fit.hq %>%
  parameterEstimates %>%
  filter(op == "=~") %>%
  formatAsTable

lhs	op	rhs	est	se	z	pvalue	ci.lower	ci.upper
aff	=~	aff1	0.93	0.02	44.16	0.00	0.89	0.97
aff	=~	aff2	0.99	0.02	64.73	0.00	0.96	1.02
aff	=~	aff3	1.06	0.02	69.58	0.00	1.03	1.09
aff	=~	aff4	1.01	0.01	69.31	0.00	0.98	1.04
cog	=~	cog1	1.67	0.07	25.64	0.00	1.54	1.80
cog	=~	cog2	1.16	0.05	21.70	0.00	1.05	1.26
cog	=~	cog3	0.98	0.06	15.37	0.00	0.85	1.10
cog	=~	cog4.r	0.20	0.08	2.39	0.02	0.04	0.36
lik	=~	lik1	1.30	0.04	35.51	0.00	1.23	1.37
lik	=~	lik2	1.08	0.03	32.75	0.00	1.01	1.14
lik	=~	lik3.r	0.63	0.05	13.82	0.00	0.54	0.72
lik	=~	lik4.r	0.99	0.04	25.89	0.00	0.92	1.07

Looks like cog4 had some marginal improvement that shows up in both the loading and the reliability.

I will run it one more time without cog4 and we’ll see if that improves things.

cfa.fit.cog <- cfa.model %>%
  str_remove(fixed(" + cog4.r")) %>%
  cfa(adhd.data.hq, effect.coding = T)

cfa.fit.cog %>%
  reliability %>%
  as_tibble(rownames = "stat") %>%
  formatAsTable

stat	aff	cog	lik
alpha	0.91	0.75	0.80
omega	0.88	0.70	0.73
omega2	0.88	0.70	0.73
omega3	0.88	0.69	0.71
avevar	0.71	0.54	0.51

# fit statistics
cfa.fit.cog %>%
  fitMeasures(fit.measures = m) %>%
  round(3) %>%
  as_tibble(rownames = 'stat') %>%
  formatAsTable

stat	value
chisq	152.61
df	41.00
pvalue	0.00
rmsea	0.08
tli	0.99
cfi	0.99

Nice. The McDonald’s \(\omega\) for cog increased to 0.7!

Save data:

adhd.data %>%
  saveRDS(file.path("..", "data", "adhd-data.rds"))

adhd.data %>%
  write_csv(file.path("..", "data", "adhd-data.csv"))

adhd.data.hq %>%
  saveRDS(file.path("..", "data", "hq-data.rds"))

adhd.data.hq %>%
  write_csv(file.path("..", "data", "hq-data.csv"))

Output document:

options(knitr.duplicate.label = "allow")
rmarkdown::render("measurement.Rmd",
                  output_dir = file.path("..", "github", "thesis"))

Measurement Assessment

5/28/2021

Inter-Item Reliabilities

Exploratory Factor Analysis

Confirmatory Factor Analysis