>>and then I was watching that it only executed when I changed my value manually (this is what I wanted).
>
>Don't think you can stop the validation running on page load.. Above looks OK to me

This did stop it from running server-side validation for the page load. It now only runs when I change the value. Here is my current code

 return {

            require: "ngModel",

            scope: {
                primaryKey: "=?",
                tableName: "@",
                columnToTest: "@",
                errorMessage: "=?",
                valueToCompare: "=?"
            },

            link: function ($scope, element, attrs, ngModel) {
                if (!ngModel) return;

                originalValue = ngModel.$modelValue;

                $scope.$watch('valueToCompare', function (valueToCompare) {
                    // watches for changes from valueToCompare binding
                    ngModel.$validate();
                });

                ngModel.$validators.codeUnique = function (modelValue, viewValue) {
                    let status = true;
                    if (_.isEmpty(viewValue.trim()) || _.isEmpty($scope.valueToCompare.trim())) {
                        status = true;
                    }
                    else {
                        if (viewValue.trim().toUpperCase() === $scope.valueToCompare.trim().toUpperCase()) {
                            status = false;
                        }
                    }
                    return status;
                };


                ngModel.$asyncValidators.smCodeUnique = function (modelValue, viewValue) {
                    if (!originalValue) {
                        originalValue = modelValue;
                    }
                    if (!viewValue || _.isEmpty(modelValue.trim()) || modelValue == originalValue) {
                        return services.Q.when(true);
                    }

                    var deferred = services.Deferred;
                    $log.info("Firing server-side validations for " + $scope.tableName + '.' + $scope.columnToTest);
                    let codeObject = {
                        id: $scope.primaryKey, tableName: $scope.tableName,
                        columnToTest: $scope.columnToTest, code: viewValue
                    };
                    return services.Http.put('api/items/checkCodeUniqueness', codeObject).then(
                        function (response) {
                            if (!response.data.isValid) {
                                $scope.errorMessage = response.data.errorMessage;
                                deferred.reject(response.data.errorMessage);
                            }
                            else {
                                deferred.resolve(response.data);
                            }
                            return deferred.promise;
                        }
                    );
                };
                //     }
            }
        };