alexf101 · April 23, 2024 23:27 · thedmeyer · Sep 30, 2021 · alexf101 · Oct 1, 2021
diff --git a/generate_dependabot_config.rb b/generate_dependabot_config.rb
 #!/usr/bin/env ruby
 require 'bundler/inline'

 gemfile do
    source 'https://rubygems.org'
    gem 'httparty'
    gem 'fast_ignore'
 end

 require 'httparty'
 require 'yaml'
 require 'set'
 require 'json'
 require 'pp'

 CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml'
 DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml'
 REPO_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
 Dir.chdir REPO_ROOT

 GLOBAL_EXCLUDE_REGEXES = [
    %r{\/node_modules\/},
 ]

 # Validates the special config file that understands Stile's extensions.
 def validate_generator_config(config_name, config)
    raise 'Missing updates?' unless config['updates'].length >= 1
    config['updates'].each do |target|
        raise 'Missing package-ecosystem' unless target['package-ecosystem']
        raise 'Missing schedule' unless target['schedule']
        raise 'Missing apply-to-glob' unless target['apply-to-glob']
        target
            .fetch('ignore', [])
            .each do |ignored_update|
                dependency_name = ignored_update['dependency-name']
                unless dependency_name
                    raise "Missing dependency name: #{target.pretty_inspect}"
                end
                unless ignored_update['blocked-reason']
                    raise "#{dependency_name}: Missing blocked-reason"
                end

                # This whitelist of valid values for blocked-reason intentionally makes it a bit difficult to add new ones willy-nilly. Please consider carefully whether whatever the new reason is, is actually a good enough reason to block updates.
                unless %w[
                           bad-upstream-version
                           deletion-planned
                           requires-other-project:drop-ie11
                           requires-other-project:delete-backbone
                       ].include?(ignored_update['blocked-reason'])
                    raise "#{dependency_name}: Invalid value for blocked-reason"
                end
                unless ignored_update['blocked-reason-detail']
                    raise "#{dependency_name}: Missing blocked-reason-detail"
                end
            end
    end
    config
 rescue StandardError
    STDERR.puts "Validation failed for input config file at `#{config_name}`"
    raise
 end

 # Validates the generated vanilla Dependabot config files (to let us test that we didn't stuff up the logic)
 def validate_generated_config(config_name, generated_config)
    raise 'Missing version' unless generated_config['version']
    raise 'Missing updates' unless generated_config['updates']
    generated_config['updates'].each do |target|
        raise 'Missing directory' unless target['directory']
        valid_keys = %w[
            package-ecosystem
            directory
            schedule
            allow
            assignees
            commit-message
            ignore
            insecure-external-code-execution
            labels
            milestone
            open-pull-requests-limit
            pull-request-branch-name
            rebase-strategy
            registries
            reviewers
            target-branch
            vendor
            versioning-strategy
        ]
        invalid_keys = target.keys - valid_keys
        if invalid_keys.length > 0
            raise "Contains invalid keys: #{invalid_keys}"
        end
        target
            .fetch('ignore', [])
            .each do |ignored_update|
                dependency_name = ignored_update['dependency-name']
                raise 'Missing dependency name' unless dependency_name
                invalid_keys =
                    ignored_update.keys - %w[
                        dependency-name
                        version_requirement
                    ]
                if invalid_keys.length > 0
                    raise "Contains invalid keys: #{invalid_keys}"
                end
            end
    end
    generated_config
 rescue StandardError
    STDERR.puts "Validation failed for config file `#{config_name}`"
    raise
 end

 # Core logic of this script - translates our extensions into a vanilla Dependabot format through judicious use of glob and regex.
 def generate_config(config_name, config_generator_config)
    config = YAML.load(config_generator_config)
    validate_generator_config(config_name, config)
    new_config = YAML.load(config_generator_config)
    new_config['updates'] = []
    config['updates'].each do |target|
        apply_to_glob = target['apply-to-glob']
        exclude_regex = target['exclude-regex']

        # Find all matching glob files, while applying gitignore rules
        matched_files =
            FastIgnore.new(
                relative: true,
                argv_rules: [apply_to_glob],
                root: REPO_ROOT,
            ).to_a.select do |path|
                !GLOBAL_EXCLUDE_REGEXES.any? do |global_exclude_regex|
                    global_exclude_regex.match?(path)
                end
            end.sort

        if exclude_regex
            exclude_regex_parsed = Regexp.new(exclude_regex)
            matched_files =
                matched_files.reject do |path|
                    exclude_regex_parsed.match?(path)
                end
        end

        if target['ignore']
            target['ignore'].each do |ignored_update|
                ignored_update.reject! do |k|
                    %w[blocked-reason blocked-reason-detail].include?(k)
                end
            end
        end
        matched_files.each do |filepath|
            new_config['updates'] <<
                {
                    # Directory is relative to the repository's root, with '/' indicating the repo's root. However, it also seems to find '.' and relative paths without '/' acceptable, so... let's just do that then.
                    # See https://dependabot.com/docs/config-file/validator/ for confirmation.
                    'directory' => File.dirname(filepath),
                }.merge(
                    target.reject do |k|
                        %w[apply-to-glob exclude-regex].include?(k)
                    end,
                )
        end

        # Remove duplicates - later entries should override earlier ones.
        found_so_far = Set.new
        new_config['updates'] =
            new_config['updates'].reverse.reject do |target|
                unique_key = [target['directory'], target['package-ecosystem']]
                should_skip = found_so_far.include?(unique_key)
                found_so_far << unique_key
                should_skip
            end.sort_by do |target|
                # Stable output important to minimise human-readable diff.
                #
                # There is no "primary key" for entries in the Dependabot
                # config schema (https://dependabot.com/docs/config-file/)
                # but it seems unlikely that we'll have multiple entries
                # that share the same directory and package manager -- e.g.
                # updating exactly the same stuff but on a different schedule.
                [target['directory'], target['package-ecosystem']]
            end
    end
    validate_generated_config(config_name, new_config)
    new_config
 end

 ### UNIT TESTS ###
 # May as well run them every time the script runs for a wee little project like this.
 cfg = generate_config('test 1', (<<~EOF))
    version: 2
    updates:
        # The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
        - package-ecosystem: "npm"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          # Apply this opt-in config by default to every project with a package.json.
          apply-to-glob: "**/package.json"
          exclude-regex: "node_modules"
 EOF

 if cfg['updates'].any? { |target| target['directory'].include?('node_modules') }
    raise "Doesn't exclude node_modules"
 end
 raise "Doesn't seem to be finding all the files" if cfg['updates'].length < 20
 cfg = generate_config('test 2', (<<~EOF))
    version: 2
    updates:
        # The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
        - package-ecosystem: "bundler"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          apply-to-glob: "**/Gemfile"
          exclude-regex: "localgems"
          ignore:
            # TODO: DRP to provide a reason for this. It was blocked for unspecified reasons in https://github.com/StileEducation/dev-environment/pull/6335.
            # UPDATE CONDITION: Requires investigation.
            - dependency-name: "mongo"
              blocked-reason: "bad-upstream-version"
              blocked-reason-detail: "We just don't like it, Sam I am"
 EOF

 if cfg['updates'].any? do |target|
       target['ignore'].first['dependency-name'] != 'mongo'
   end
    raise "Doesn't keep ignore"
 end

 cfg = generate_config('test 3', (<<~EOF))
    version: 2
    updates:
        # The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
        - package-ecosystem: "npm"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          # Apply this opt-in config by default to every project with a package.json.
          apply-to-glob: "**/package.json"
          exclude-regex: "node_modules"
        - package-ecosystem: "npm"
          # The web-client itself has a lot of special-case restrictions.
          apply-to-glob: "web-client/package.json"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
          ignore:
              # UPDATE CONDITION: IE11 EOL.
              - dependency-name: "uuid"
                blocked-reason: "bad-upstream-version"
                blocked-reason-detail: "Breaks IE11 for incomprehensible reasons"
        - package-ecosystem: "docker"
          apply-to-glob: "web-client/Dockerfile"
          schedule:
            interval: "daily"
            time: "02:00"
            timezone: "Australia/Melbourne"
 EOF

 unless cfg['updates'].any? do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'npm'
       end
    raise 'Docker package stomped JavaScript package'
 end
 unless cfg['updates'].any? do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'docker'
       end
    raise 'Web-client package stomped Docker package'
 end
 unless cfg['updates'].select do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'npm'
       end.length == 1
    raise 'Found duplicate entries for the same directory and package manager'
 end
 unless cfg['updates'].select do |target|
           target['directory'] == 'web-client' &&
               target['package-ecosystem'] == 'npm'
       end.first[
           'ignore'
       ].first[
           'dependency-name'
       ] == 'uuid'
    raise "Preserved directory isn't the later one"
 end

 # While developing this script, you may find it useful to uncomment the following line to only run the tests.
 # puts "PASSED TESTS"; exit 0
 # For bonus points, feel free to run in 'watch mode' using the following wizardly incantation :)
 # fswatch generate_dependabot_config.rb | xargs -n1 -I{} generate_dependabot_config.rb

 ### ACTUAL SCRIPT INVOCATION ###

 generated_config =
    generate_config(
        CONFIG_GENERATOR_CONFIG_FILENAME,
        File.read(CONFIG_GENERATOR_CONFIG_FILENAME),
    )

 File.open(DEPENDABOT_ACTUAL_CONFIG_FILENAME, 'w') do |f|
    f.puts(
        '# DO NOT MODIFY THIS FILE BY HAND, IT IS GENERATED BY bin/generate_dependabot_config.rb',
    )
    f.puts(
        YAML.dump(
            # Why, you may wonder, do I convert this generated_config to json and then parse it... before outputting it as yaml?
            # No, I don't serialize things in multiple formats and back just for fun. The Ruby YAML output will 'cleverly' use advanced YAML syntax, 'aliases',
            # to represent objects with the same identity in memory. Neat as this is, the Dependabot validator says it doesn't support that syntax.
            # This re-serialisation avoids that issue by forcing Ruby to make different objects in memory for every entry.
            JSON.parse(generated_config.to_json),
        ),
    )
 end
 puts "Wrote out new dependabot config to #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}"
 puts 'Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/'

 puts "Checking that the config we generated passes Dependabot's validator API"
 response =
    HTTParty.post(
        'https://api.dependabot.com/config_files/validate',
        body: {
            'config-file-body': generated_config.to_json,
        },
    )
 pp response.parsed_response
 unless response.parsed_response['errors'].empty?
    raise 'Dependabot says this config is invalid'
 end

 # If you use prettier, you may want to uncomment the following lines :)
 # begin
 #     `prettier --write #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}`
 # rescue => e
 #     puts "Unable to run prettier: #{e.inspect}"
 # end
	#!/usr/bin/env ruby
	require 'bundler/inline'

	gemfile do
	source 'https://rubygems.org'
	gem 'httparty'
	gem 'fast_ignore'
	end

	require 'httparty'
	require 'yaml'
	require 'set'
	require 'json'
	require 'pp'

	CONFIG_GENERATOR_CONFIG_FILENAME = 'dependabot_config_generator_config.yml'
	DEPENDABOT_ACTUAL_CONFIG_FILENAME = '.github/dependabot.yml'
	REPO_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
	Dir.chdir REPO_ROOT

	GLOBAL_EXCLUDE_REGEXES = [
	%r{\/node_modules\/},
	]

	# Validates the special config file that understands Stile's extensions.
	def validate_generator_config(config_name, config)
	raise 'Missing updates?' unless config['updates'].length >= 1
	config['updates'].each do \|target\|
	raise 'Missing package-ecosystem' unless target['package-ecosystem']
	raise 'Missing schedule' unless target['schedule']
	raise 'Missing apply-to-glob' unless target['apply-to-glob']
	target
	.fetch('ignore', [])
	.each do \|ignored_update\|
	dependency_name = ignored_update['dependency-name']
	unless dependency_name
	raise "Missing dependency name: #{target.pretty_inspect}"
	end
	unless ignored_update['blocked-reason']
	raise "#{dependency_name}: Missing blocked-reason"
	end

	# This whitelist of valid values for blocked-reason intentionally makes it a bit difficult to add new ones willy-nilly. Please consider carefully whether whatever the new reason is, is actually a good enough reason to block updates.
	unless %w[
	bad-upstream-version
	deletion-planned
	requires-other-project:drop-ie11
	requires-other-project:delete-backbone
	].include?(ignored_update['blocked-reason'])
	raise "#{dependency_name}: Invalid value for blocked-reason"
	end
	unless ignored_update['blocked-reason-detail']
	raise "#{dependency_name}: Missing blocked-reason-detail"
	end
	end
	end
	config
	rescue StandardError
	STDERR.puts "Validation failed for input config file at `#{config_name}`"
	raise
	end

	# Validates the generated vanilla Dependabot config files (to let us test that we didn't stuff up the logic)
	def validate_generated_config(config_name, generated_config)
	raise 'Missing version' unless generated_config['version']
	raise 'Missing updates' unless generated_config['updates']
	generated_config['updates'].each do \|target\|
	raise 'Missing directory' unless target['directory']
	valid_keys = %w[
	package-ecosystem
	directory
	schedule
	allow
	assignees
	commit-message
	ignore
	insecure-external-code-execution
	labels
	milestone
	open-pull-requests-limit
	pull-request-branch-name
	rebase-strategy
	registries
	reviewers
	target-branch
	vendor
	versioning-strategy
	]
	invalid_keys = target.keys - valid_keys
	if invalid_keys.length > 0
	raise "Contains invalid keys: #{invalid_keys}"
	end
	target
	.fetch('ignore', [])
	.each do \|ignored_update\|
	dependency_name = ignored_update['dependency-name']
	raise 'Missing dependency name' unless dependency_name
	invalid_keys =
	ignored_update.keys - %w[
	dependency-name
	version_requirement
	]
	if invalid_keys.length > 0
	raise "Contains invalid keys: #{invalid_keys}"
	end
	end
	end
	generated_config
	rescue StandardError
	STDERR.puts "Validation failed for config file `#{config_name}`"
	raise
	end

	# Core logic of this script - translates our extensions into a vanilla Dependabot format through judicious use of glob and regex.
	def generate_config(config_name, config_generator_config)
	config = YAML.load(config_generator_config)
	validate_generator_config(config_name, config)
	new_config = YAML.load(config_generator_config)
	new_config['updates'] = []
	config['updates'].each do \|target\|
	apply_to_glob = target['apply-to-glob']
	exclude_regex = target['exclude-regex']

	# Find all matching glob files, while applying gitignore rules
	matched_files =
	FastIgnore.new(
	relative: true,
	argv_rules: [apply_to_glob],
	root: REPO_ROOT,
	).to_a.select do \|path\|
	!GLOBAL_EXCLUDE_REGEXES.any? do \|global_exclude_regex\|
	global_exclude_regex.match?(path)
	end
	end.sort

	if exclude_regex
	exclude_regex_parsed = Regexp.new(exclude_regex)
	matched_files =
	matched_files.reject do \|path\|
	exclude_regex_parsed.match?(path)
	end
	end

	if target['ignore']
	target['ignore'].each do \|ignored_update\|
	ignored_update.reject! do \|k\|
	%w[blocked-reason blocked-reason-detail].include?(k)
	end
	end
	end
	matched_files.each do \|filepath\|
	new_config['updates'] <<
	{
	# Directory is relative to the repository's root, with '/' indicating the repo's root. However, it also seems to find '.' and relative paths without '/' acceptable, so... let's just do that then.
	# See https://dependabot.com/docs/config-file/validator/ for confirmation.
	'directory' => File.dirname(filepath),
	}.merge(
	target.reject do \|k\|
	%w[apply-to-glob exclude-regex].include?(k)
	end,
	)
	end

	# Remove duplicates - later entries should override earlier ones.
	found_so_far = Set.new
	new_config['updates'] =
	new_config['updates'].reverse.reject do \|target\|
	unique_key = [target['directory'], target['package-ecosystem']]
	should_skip = found_so_far.include?(unique_key)
	found_so_far << unique_key
	should_skip
	end.sort_by do \|target\|
	# Stable output important to minimise human-readable diff.
	#
	# There is no "primary key" for entries in the Dependabot
	# config schema (https://dependabot.com/docs/config-file/)
	# but it seems unlikely that we'll have multiple entries
	# that share the same directory and package manager -- e.g.
	# updating exactly the same stuff but on a different schedule.
	[target['directory'], target['package-ecosystem']]
	end
	end
	validate_generated_config(config_name, new_config)
	new_config
	end

	### UNIT TESTS ###
	# May as well run them every time the script runs for a wee little project like this.
	cfg = generate_config('test 1', (<<~EOF))
	version: 2
	updates:
	# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
	- package-ecosystem: "npm"
	schedule:
	interval: "daily"
	time: "02:00"
	timezone: "Australia/Melbourne"
	# Apply this opt-in config by default to every project with a package.json.
	apply-to-glob: "**/package.json"
	exclude-regex: "node_modules"
	EOF

	if cfg['updates'].any? { \|target\| target['directory'].include?('node_modules') }
	raise "Doesn't exclude node_modules"
	end
	raise "Doesn't seem to be finding all the files" if cfg['updates'].length < 20
	cfg = generate_config('test 2', (<<~EOF))
	version: 2
	updates:
	# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
	- package-ecosystem: "bundler"
	schedule:
	interval: "daily"
	time: "02:00"
	timezone: "Australia/Melbourne"
	apply-to-glob: "**/Gemfile"
	exclude-regex: "localgems"
	ignore:
	# TODO: DRP to provide a reason for this. It was blocked for unspecified reasons in https://github.com/StileEducation/dev-environment/pull/6335.
	# UPDATE CONDITION: Requires investigation.
	- dependency-name: "mongo"
	blocked-reason: "bad-upstream-version"
	blocked-reason-detail: "We just don't like it, Sam I am"
	EOF

	if cfg['updates'].any? do \|target\|
	target['ignore'].first['dependency-name'] != 'mongo'
	end
	raise "Doesn't keep ignore"
	end

	cfg = generate_config('test 3', (<<~EOF))
	version: 2
	updates:
	# The vast majority of our NodeJS code ought to receive automatic updates, and isn't subject to the same burdensome constrictions that the main web-client bundle is (e.g. Node 8).
	- package-ecosystem: "npm"
	schedule:
	interval: "daily"
	time: "02:00"
	timezone: "Australia/Melbourne"
	# Apply this opt-in config by default to every project with a package.json.
	apply-to-glob: "**/package.json"
	exclude-regex: "node_modules"
	- package-ecosystem: "npm"
	# The web-client itself has a lot of special-case restrictions.
	apply-to-glob: "web-client/package.json"
	schedule:
	interval: "daily"
	time: "02:00"
	timezone: "Australia/Melbourne"
	ignore:
	# UPDATE CONDITION: IE11 EOL.
	- dependency-name: "uuid"
	blocked-reason: "bad-upstream-version"
	blocked-reason-detail: "Breaks IE11 for incomprehensible reasons"
	- package-ecosystem: "docker"
	apply-to-glob: "web-client/Dockerfile"
	schedule:
	interval: "daily"
	time: "02:00"
	timezone: "Australia/Melbourne"
	EOF

	unless cfg['updates'].any? do \|target\|
	target['directory'] == 'web-client' &&
	target['package-ecosystem'] == 'npm'
	end
	raise 'Docker package stomped JavaScript package'
	end
	unless cfg['updates'].any? do \|target\|
	target['directory'] == 'web-client' &&
	target['package-ecosystem'] == 'docker'
	end
	raise 'Web-client package stomped Docker package'
	end
	unless cfg['updates'].select do \|target\|
	target['directory'] == 'web-client' &&
	target['package-ecosystem'] == 'npm'
	end.length == 1
	raise 'Found duplicate entries for the same directory and package manager'
	end
	unless cfg['updates'].select do \|target\|
	target['directory'] == 'web-client' &&
	target['package-ecosystem'] == 'npm'
	end.first[
	'ignore'
	].first[
	'dependency-name'
	] == 'uuid'
	raise "Preserved directory isn't the later one"
	end

	# While developing this script, you may find it useful to uncomment the following line to only run the tests.
	# puts "PASSED TESTS"; exit 0
	# For bonus points, feel free to run in 'watch mode' using the following wizardly incantation :)
	# fswatch generate_dependabot_config.rb \| xargs -n1 -I{} generate_dependabot_config.rb

	### ACTUAL SCRIPT INVOCATION ###

	generated_config =
	generate_config(
	CONFIG_GENERATOR_CONFIG_FILENAME,
	File.read(CONFIG_GENERATOR_CONFIG_FILENAME),
	)

	File.open(DEPENDABOT_ACTUAL_CONFIG_FILENAME, 'w') do \|f\|
	f.puts(
	'# DO NOT MODIFY THIS FILE BY HAND, IT IS GENERATED BY bin/generate_dependabot_config.rb',
	)
	f.puts(
	YAML.dump(
	# Why, you may wonder, do I convert this generated_config to json and then parse it... before outputting it as yaml?
	# No, I don't serialize things in multiple formats and back just for fun. The Ruby YAML output will 'cleverly' use advanced YAML syntax, 'aliases',
	# to represent objects with the same identity in memory. Neat as this is, the Dependabot validator says it doesn't support that syntax.
	# This re-serialisation avoids that issue by forcing Ruby to make different objects in memory for every entry.
	JSON.parse(generated_config.to_json),
	),
	)
	end
	puts "Wrote out new dependabot config to #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}"
	puts 'Please check that Dependabot thinks its valid using https://dependabot.com/docs/config-file/validator/'

	puts "Checking that the config we generated passes Dependabot's validator API"
	response =
	HTTParty.post(
	'https://api.dependabot.com/config_files/validate',
	body: {
	'config-file-body': generated_config.to_json,
	},
	)
	pp response.parsed_response
	unless response.parsed_response['errors'].empty?
	raise 'Dependabot says this config is invalid'
	end

	# If you use prettier, you may want to uncomment the following lines :)
	# begin
	# `prettier --write #{DEPENDABOT_ACTUAL_CONFIG_FILENAME}`
	# rescue => e
	# puts "Unable to run prettier: #{e.inspect}"
	# end
No results found