% SPDX-License-Identifier: AGPL-3.0-or-later
% Copyright (C) 2026 William Bert Craytor. Published by Valuation Engineer
% Research Labs (Pacific Vista Net). This program is free software: you can
% redistribute it and/or modify it under the terms of the GNU Affero General
% Public License as published by the Free Software Foundation, either version 3
% of the License, or (at your option) any later version. It is distributed
% WITHOUT ANY WARRANTY, without even the implied warranty of MERCHANTABILITY or
% FITNESS FOR A PARTICULAR PURPOSE; see
% <https://www.gnu.org/licenses/agpl-3.0.html>.

%% property-variables-endogeneity.pl
%%
%% Prolog companion to the working paper
%%   "Property Variables Most Subject to Endogeneity" (Research Labs, 2026)
%%   Author:  Bert Craytor
%%   Version: 1.0.0 (draft)
%%
%% It states the logic of the paper as facts and rules, so that a reader can
%% query it. It builds on the vocabulary of Foundations entry
%% 009 (Endogeneity), Vol. 1 No. 2, version 1.1.0, without loading it: this
%% file runs on its own. It runs under Trealla (the Journal's official
%% Prolog), SWI-Prolog and Scryer Prolog.
%%
%% Try:
%%   ?- run_tests.
%%   ?- often_endogenous_via(living_area, Cause).
%%   ?- matters_for(days_on_market, Use).
%%   ?- confounding_part(0.75, 90000, X).
%%   ?- cqa_score(12000, [-30000,-8000,0,5000,15000,22000], S).

:- use_module(library(lists)).
:- dynamic(moves_with_structural_error/3).   % filled in for a given model

% ---------------------------------------------------------------------------
% 1. The method assumed
% ---------------------------------------------------------------------------

method_assumed(residual_constraint_approach).

% Features the records measure, and those they do not (latent).
measured(living_area).
measured(age).
measured(lot_size).
measured(bedrooms).
measured(bathrooms).

latent(condition).
latent(quality).
latent(appeal).
latent(unrecorded_location).
latent(other_unmeasured_features).

% Only measured features are eligible for the primary regression, the one
% from which the value conclusion is reached; MARS selects among them. The
% CQA score and the appraiser's ratings of latent features never enter it.
eligible_for_primary_regression(F) :- measured(F).

% What the appraiser's ratings of latent features are for.
purpose_of_latent_ratings(explain_residual_differences_between_properties).
affects_value_conclusion(latent_ratings, no).

% The residual of a comparable sale and its components.
residual_definition(residual = net_sale_price - primary_regression_estimate).
net_sale_price_definition(net_sale_price = sale_price - concessions).
residual_component(cqa,       [condition, quality, appeal]).
residual_component(omitted,   [unrecorded_location, other_unmeasured_features]).
residual_component(eta,       [model_error, negotiation_noise]).

% ---------------------------------------------------------------------------
% 2. The definition (entry 009, Endogeneity) and the scrutiny test
% ---------------------------------------------------------------------------

% A regressor is endogenous, relative to a model and a causal question,
% when it moves with the structural error u.
endogenous(Regressor, Model, Question) :-
    moves_with_structural_error(Regressor, Model, Question).

% The test that recurs through the paper. A regressor deserves scrutiny
% when someone chose its value or measured it using price-relevant
% information the model does not have. Selection is a property of the
% sales as a whole, not of one regressor: they deserve the same scrutiny
% when what put a house among them is connected to its price.
scrutiny_reason(regressor, chosen).
scrutiny_reason(regressor, measured).
scrutiny_reason(sales_as_a_whole, owners_decision_to_sell).
scrutiny_reason(sales_as_a_whole, criteria_for_choosing_the_sales).

% ---------------------------------------------------------------------------
% 3. When endogeneity affects the sale price and market value
% ---------------------------------------------------------------------------

affects_value(reading_a_coefficient_as_an_adjustment).
affects_value(counterfactual_value).
affects_value(subject_unlike_comparables_in_the_driving_respect).

% Market value of the property as it is: the target is the conditional
% expectation (entry 007, Conditional Expectation), which a model can
% estimate well even when its coefficients are not causal effects.
unaffected_target(market_value_as_is, conditional_expectation,
                  provided(subject_features_covered_by_comparables)).

% ---------------------------------------------------------------------------
% 4. Actions and states
% ---------------------------------------------------------------------------

variable_kind(recent_remodel,           action).
variable_kind(recent_updates,           action).
variable_kind(addition,                 action).
variable_kind(accessory_dwelling_unit,  action).
variable_kind(pool,                     action).
variable_kind(solar,                    action).
variable_kind(condition,                state).
variable_kind(quality,                  state).
variable_kind(functional_utility,       state).
variable_kind(living_area,              state).
variable_kind(age,                      state).

changes(recent_remodel, condition).
changes(recent_remodel, quality).
changes(recent_updates, condition).
changes(recent_updates, quality).

% How each kind usually becomes endogenous.
common_route(action, owner_choice).
common_route(state,  measurement).
common_route(state,  owner_choice).        % living area is partly chosen

% An action is evidence about the part of the state the ratings miss, and
% work done to code with inspections passed documents safety and soundness.
evidence_about(Action, State) :- changes(Action, State).
documents(Action, safety_and_soundness) :-
    variable_kind(Action, action),
    assumed(code_followed), assumed(required_inspections_approved).

assumed(code_followed).
assumed(required_inspections_approved).

% ---------------------------------------------------------------------------
% 5. Roles and the inclusion rule (entry 009, Endogeneity)
% ---------------------------------------------------------------------------

role(remodel,          treatment).
role(condition,        mediator).
role(location_quality, confounder).
role(price,            outcome).

% Whether to include a variable depends on the effect wanted.
include_rule(confounder, _AnyEffect,   yes).
include_rule(mediator,   total_effect, no).
include_rule(mediator,   direct_effect, yes).

include_variable(V, Effect, Answer) :-
    role(V, Role), include_rule(Role, Effect, Answer).

% ---------------------------------------------------------------------------
% 6. The variables, by cause (the paper's summary table)
% ---------------------------------------------------------------------------

often_endogenous_via(recent_remodel,       owner_choice).
often_endogenous_via(recent_updates,       owner_choice).
often_endogenous_via(addition,             owner_choice).
often_endogenous_via(accessory_dwelling_unit, owner_choice).
often_endogenous_via(pool,                 owner_choice).
often_endogenous_via(solar,                owner_choice).
often_endogenous_via(living_area,          owner_choice).
often_endogenous_via(living_area,          record_error).
often_endogenous_via(age,                  what_happened_since_construction).
often_endogenous_via(list_price,           set_with_price_relevant_information).
often_endogenous_via(concessions,          determined_with_price).
often_endogenous_via(seller_credits,       determined_with_price).
often_endogenous_via(financing_terms,      determined_with_price).  % some
often_endogenous_via(days_on_market,       outcome_of_marketing).
often_endogenous_via(sale_type_and_timing, selection).

% School district and neighborhood: a proper location adjustment for the
% market value as is; unrecorded traits travel with the district, which
% matters only for questions about the schools themselves.
awareness(school_district, unrecorded_traits_travel_with_it).
matters_only_for(school_district, questions_about_the_schools_themselves).

% Physically fixed: not caused by the price, but not necessarily exogenous;
% such a feature can still travel with features the model omits. A view
% across land a neighbor could build on also carries an unrecorded risk of
% impairment, and is not the same feature as a view protected by terrain.
physically_fixed(position).
physically_fixed(distance).
physically_fixed(terrain_fixed_view).
physically_fixed(slope).
carries_unrecorded_risk(view_across_buildable_land, impairment).

usual_direction(recent_remodel, owner_choice,
                up_where_owners_invest_in_better_houses).
usual_direction(living_area, owner_choice,
                up_if_size_travels_with_unrecorded_location).
usual_direction(living_area, record_error,
                either_way_not_shown_by_the_sales).
usual_direction(age, what_happened_since_construction, varies_by_market).
usual_direction(sale_type_and_timing, selection,
                level_up_rates_move_if_selection_depends_on_features).

matters_for(V, adjustment) :-
    often_endogenous_via(V, C), first_cause(V, C), C \== selection,
    \+ transaction_variable(V).
matters_for(V, counterfactual_value) :-
    variable_kind(V, action), often_endogenous_via(V, owner_choice).
matters_for(V, not_a_cause_of_value) :- transaction_variable(V).
matters_for(sale_type_and_timing, every_use).

first_cause(V, C) :- often_endogenous_via(V, C1), !, C = C1.


transaction_variable(list_price).
transaction_variable(concessions).
transaction_variable(seller_credits).
transaction_variable(financing_terms).
transaction_variable(days_on_market).

% ---------------------------------------------------------------------------
% 7. Measurement error
% ---------------------------------------------------------------------------

% Recorded living area: MLS and assessor errors vary in sign and size by
% locality, so their effect on the living-area rate, and on the rates of
% features that move with living area, cannot be read from the sales.
record_error_effect(living_area, direction_not_shown_by_the_sales).
reaches_rates_of(living_area, [location, lot_size, bedrooms, bathrooms]).

% Record error is not removed by adding comparables.
removed_by_more_comparables(record_error, no).

% ---------------------------------------------------------------------------
% 8. The confounding part (Theil's formula, entry 008, Omitted Variable
%    Bias; exact in the linear specification)
% ---------------------------------------------------------------------------

% Gamma: price added by one unit of the unrecorded feature.
% Delta: how much more of it the remodeled houses have.
confounding_part(Delta, Gamma, Part) :- Part is Gamma * Delta.

% Gain from remodeling, given the observed coefficient and the assumptions.
gain(Coefficient, Delta, Gamma, Gain) :-
    confounding_part(Delta, Gamma, Part),
    Gain is Coefficient - Part.

% The paper's grid (observed coefficient from the simulated market).
observed_remodel_coefficient(144792).
grid_delta(0.25). grid_delta(0.50). grid_delta(0.75). grid_delta(1.00).
grid_gamma(30000). grid_gamma(60000). grid_gamma(90000). grid_gamma(120000).

grid_cell(Delta, Gamma, Gain) :-
    observed_remodel_coefficient(B),
    grid_delta(Delta), grid_gamma(Gamma),
    G0 is B - Gamma * Delta, Gain is round(G0).

% The grid is a practitioner heuristic: a range of stated assumptions for
% one omitted feature acting linearly, not a formal bound
% (Cinelli and Hazlett 2020 give the formal bounds).
grid_is(stated_assumptions).
grid_is_not(formal_bound).

% ---------------------------------------------------------------------------
% 9. The CQA score and the placement of the subject
% ---------------------------------------------------------------------------

ranking_basis(residual).
ranking_basis(residual_per_square_foot).   % the appraiser chooses

% CQA score: percent of the market area's sales with lower residuals, / 10.
cqa_score(SubjectResidual, Residuals, Score) :-
    count_lower(Residuals, SubjectResidual, Lower),
    length(Residuals, N), N > 0,
    Score is 10 * Lower / N.

count_lower([], _, 0).
count_lower([R|Rs], X, C) :-
    count_lower(Rs, X, C0),
    ( R < X -> C is C0 + 1 ; C = C0 ).

placement_evidence(mls_data).
placement_evidence(mls_photographs).

placement_rule(1, cqa,
    place_on_all_observed_evidence_of_latent_state_not_on_what_travels_with_it).
placement_rule(2, measured,
    place_on_latent_value_beyond_what_the_measured_features_carry).

placement_check(other_appraisers_in_the_area_find_it_reasonable).

% ---------------------------------------------------------------------------
% 10. Dividing the residual does not change the conclusion (RCA Proof II)
% ---------------------------------------------------------------------------

% A division of a residual is a list of contributions that sum to it.
valid_division(Residual, Parts) :- sum(Parts, S), S =:= Residual.

% Total residual adjustment for a comparable: subject's residual minus the
% comparable's, whatever the division.
residual_adjustment(SubjectParts, CompParts, Adjustment) :-
    sum(SubjectParts, S), sum(CompParts, C),
    Adjustment is S - C.

sum([], 0).
sum([X|Xs], S) :- sum(Xs, S0), S is S0 + X.

% ---------------------------------------------------------------------------
% 11. What a report should state
% ---------------------------------------------------------------------------

report_item(1, question_each_coefficient_answers).
report_item(2,
    for_each_adjustment_whether_listed_and_likely_direction_of_error).
report_item(3, counterfactual_value_two_assumptions_ranges_and_value_over_them).
report_item(4, basis_of_subject_placement_mls_data_and_photographs).

% ---------------------------------------------------------------------------
% Self-test
% ---------------------------------------------------------------------------

test(rca_assumed,           method_assumed(residual_constraint_approach)).
test(cqa_not_in_regression, \+ eligible_for_primary_regression(cqa_score)).
test(latent_not_in_regression,
     \+ (latent(F), eligible_for_primary_regression(F))).
test(ratings_explanatory,   affects_value_conclusion(latent_ratings, no)).
test(remodel_is_action,     variable_kind(recent_remodel, action)).
test(condition_is_state,    variable_kind(condition, state)).
test(include_confounder,
     include_variable(location_quality, total_effect, yes)).
test(exclude_mediator,      include_variable(condition, total_effect, no)).
test(living_area_two_causes,
     ( often_endogenous_via(living_area, owner_choice),
       often_endogenous_via(living_area, record_error) )).
test(record_error_direction,
     record_error_effect(living_area, direction_not_shown_by_the_sales)).
test(no_cancelling,         removed_by_more_comparables(record_error, no)).
test(theil_product,
     ( confounding_part(0.735, 90904, P), abs(P - 66814.44) < 0.01 )).
test(grid_cell,             grid_cell(0.75, 90000, 77292)).
test(cqa_score,
     ( cqa_score(12000, [-30000,-8000,0,5000,15000,22000], S),
       abs(S - 6.6667) < 0.001 )).
test(proof_ii,
     ( valid_division(40000, [25000, 10000, 5000]),
       valid_division(40000, [10000, 20000, 10000]),
       valid_division(15000, [9000, 6000]),
       valid_division(15000, [3000, 12000]),
       residual_adjustment([25000, 10000, 5000], [9000, 6000], A1),
       residual_adjustment([10000, 20000, 10000], [3000, 12000], A2),
       A1 =:= A2 )).
test(two_rules,
     ( placement_rule(2, measured, _), \+ placement_rule(3, _, _) )).
test(one_answer_per_use,
     ( findall(U, matters_for(living_area, U), Us), Us == [adjustment] )).
test(pool_is_action,        matters_for(pool, counterfactual_value)).
test(financing_terms,       transaction_variable(financing_terms)).
test(not_a_cause,           matters_for(days_on_market, not_a_cause_of_value)).
test(district_awareness,
     matters_only_for(school_district, questions_about_the_schools_themselves)).
test(physically_fixed,      physically_fixed(terrain_fixed_view)).
test(view_risk,
     carries_unrecorded_risk(view_across_buildable_land, impairment)).

run_tests :-
    findall(Name, test(Name, _), Names),
    run_each(Names, 0, Failed),
    length(Names, N),
    Passed is N - Failed,
    write(Passed), write(' of '), write(N), write(' tests passed.'), nl,
    Failed =:= 0.

run_each([], F, F).
run_each([Name|Ns], F0, F) :-
    test(Name, Goal),
    ( call(Goal) -> F1 = F0
    ; write('FAIL: '), write(Name), nl, F1 is F0 + 1 ),
    run_each(Ns, F1, F).