diff --git a/CHANGELOG.md b/CHANGELOG.md index c117834d4..2c0375c7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## Current +## Release v1.3.4 (10 August 2024) + +* (#1060) Fix bug with return value of `Concurrent.available_processor_count` when `cpu.cfs_quota_us` is -1. +* (#1058) Add `Concurrent.cpu_shares` that is cgroups aware. + ## Release v1.3.3 (9 June 2024) * (#1053) Improve the speed of `Concurrent.physical_processor_count` on Windows. diff --git a/Gemfile b/Gemfile index f10dc8eba..1786c8b21 100644 --- a/Gemfile +++ b/Gemfile @@ -1,14 +1,14 @@ source 'https://rubygems.org' -require File.join(File.dirname(__FILE__), 'lib/concurrent-ruby/concurrent/version') -require File.join(File.dirname(__FILE__ ), 'lib/concurrent-ruby-edge/concurrent/edge/version') +version = File.read("#{__dir__}/lib/concurrent-ruby/concurrent/version.rb")[/'(.+)'/, 1] or raise +edge_version = File.read("#{__dir__}/lib/concurrent-ruby-edge/concurrent/edge/version.rb")[/'(.+)'/, 1] or raise no_path = ENV['NO_PATH'] options = no_path ? {} : { path: '.' } -gem 'concurrent-ruby', Concurrent::VERSION, options -gem 'concurrent-ruby-edge', Concurrent::EDGE_VERSION, options -gem 'concurrent-ruby-ext', Concurrent::VERSION, options.merge(platform: :mri) +gem 'concurrent-ruby', version, options +gem 'concurrent-ruby-edge', edge_version, options +gem 'concurrent-ruby-ext', version, options.merge(platform: :mri) group :development do gem 'rake', '~> 13.0' diff --git a/Rakefile b/Rakefile index 006fcb993..c52b5640e 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,5 @@ -require_relative 'lib/concurrent-ruby/concurrent/version' -require_relative 'lib/concurrent-ruby-edge/concurrent/edge/version' -require_relative 'lib/concurrent-ruby/concurrent/utility/engine' +version = File.read("#{__dir__}/lib/concurrent-ruby/concurrent/version.rb")[/'(.+)'/, 1] or raise +edge_version = File.read("#{__dir__}/lib/concurrent-ruby-edge/concurrent/edge/version.rb")[/'(.+)'/, 1] or raise core_gemspec = Gem::Specification.load File.join(__dir__, 'concurrent-ruby.gemspec') ext_gemspec = Gem::Specification.load File.join(__dir__, 'concurrent-ruby-ext.gemspec') @@ -8,14 +7,14 @@ edge_gemspec = Gem::Specification.load File.join(__dir__, 'concurrent-ruby-edge. require 'rake/javaextensiontask' -ENV['JRUBY_HOME'] = ENV['CONCURRENT_JRUBY_HOME'] if ENV['CONCURRENT_JRUBY_HOME'] && !Concurrent.on_jruby? +ENV['JRUBY_HOME'] = ENV['CONCURRENT_JRUBY_HOME'] if ENV['CONCURRENT_JRUBY_HOME'] && RUBY_ENGINE != 'jruby' Rake::JavaExtensionTask.new('concurrent_ruby', core_gemspec) do |ext| ext.ext_dir = 'ext/concurrent-ruby' ext.lib_dir = 'lib/concurrent-ruby/concurrent' end -unless Concurrent.on_jruby? || Concurrent.on_truffleruby? +if RUBY_ENGINE == 'ruby' require 'rake/extensiontask' Rake::ExtensionTask.new('concurrent_ruby_ext', ext_gemspec) do |ext| @@ -68,7 +67,7 @@ require 'rubygems' require 'rubygems/package_task' Gem::PackageTask.new(core_gemspec) {} if core_gemspec -Gem::PackageTask.new(ext_gemspec) {} if ext_gemspec && !Concurrent.on_jruby? +Gem::PackageTask.new(ext_gemspec) {} if ext_gemspec && RUBY_ENGINE != 'jruby' Gem::PackageTask.new(edge_gemspec) {} if edge_gemspec CLEAN.include( @@ -96,9 +95,9 @@ begin task :installed do Bundler.with_original_env do Dir.chdir(__dir__) do - sh "gem install pkg/concurrent-ruby-#{Concurrent::VERSION}.gem" - sh "gem install pkg/concurrent-ruby-ext-#{Concurrent::VERSION}.gem" if Concurrent.on_cruby? - sh "gem install pkg/concurrent-ruby-edge-#{Concurrent::EDGE_VERSION}.gem" + sh "gem install pkg/concurrent-ruby-#{version}.gem" + sh "gem install pkg/concurrent-ruby-ext-#{version}.gem" if RUBY_ENGINE == 'ruby' + sh "gem install pkg/concurrent-ruby-edge-#{edge_version}.gem" ENV['NO_PATH'] = 'true' sh 'bundle update' sh 'bundle exec rake spec:ci' @@ -128,7 +127,7 @@ rescue LoadError => e puts 'RSpec is not installed, skipping test task definitions: ' + e.message end -current_yard_version_name = Concurrent::VERSION +current_yard_version_name = version begin require 'yard' @@ -232,6 +231,8 @@ namespace :release do # Depends on environment of @pitr-ch task :checks do + raise '$CONCURRENT_JRUBY_HOME must be set' unless ENV['CONCURRENT_JRUBY_HOME'] + Dir.chdir(__dir__) do sh 'test -z "$(git status --porcelain)"' do |ok, res| unless ok @@ -262,6 +263,8 @@ namespace :release do desc '* test actual installed gems instead of cloned repository on MRI and JRuby' task :test do + raise '$CONCURRENT_JRUBY_HOME must be set' unless ENV['CONCURRENT_JRUBY_HOME'] + Dir.chdir(__dir__) do puts "Testing with the installed gem" @@ -310,21 +313,21 @@ namespace :release do desc '** tag HEAD with current version and push to github' task :tag => :ask do Dir.chdir(__dir__) do - sh "git tag v#{Concurrent::VERSION}" if publish_base - sh "git push origin v#{Concurrent::VERSION}" if publish_base - sh "git tag edge-v#{Concurrent::EDGE_VERSION}" if publish_edge - sh "git push origin edge-v#{Concurrent::EDGE_VERSION}" if publish_edge + sh "git tag v#{version}" if publish_base + sh "git push origin v#{version}" if publish_base + sh "git tag edge-v#{edge_version}" if publish_edge + sh "git push origin edge-v#{edge_version}" if publish_edge end end desc '** push all *.gem files to rubygems' task :rubygems => :ask do Dir.chdir(__dir__) do - sh "gem push pkg/concurrent-ruby-#{Concurrent::VERSION}.gem" if publish_base - sh "gem push pkg/concurrent-ruby-edge-#{Concurrent::EDGE_VERSION}.gem" if publish_edge - sh "gem push pkg/concurrent-ruby-ext-#{Concurrent::VERSION}.gem" if publish_base - sh "gem push pkg/concurrent-ruby-ext-#{Concurrent::VERSION}-x64-mingw32.gem" if publish_base - sh "gem push pkg/concurrent-ruby-ext-#{Concurrent::VERSION}-x86-mingw32.gem" if publish_base + sh "gem push pkg/concurrent-ruby-#{version}.gem" if publish_base + sh "gem push pkg/concurrent-ruby-edge-#{edge_version}.gem" if publish_edge + sh "gem push pkg/concurrent-ruby-ext-#{version}.gem" if publish_base + sh "gem push pkg/concurrent-ruby-ext-#{version}-x64-mingw32.gem" if publish_base + sh "gem push pkg/concurrent-ruby-ext-#{version}-x86-mingw32.gem" if publish_base end end diff --git a/concurrent-ruby-edge.gemspec b/concurrent-ruby-edge.gemspec index 4eedb53f0..719548b95 100644 --- a/concurrent-ruby-edge.gemspec +++ b/concurrent-ruby-edge.gemspec @@ -1,11 +1,11 @@ -require File.join(File.dirname(__FILE__ ), 'lib/concurrent-ruby/concurrent/version') -require File.join(File.dirname(__FILE__ ), 'lib/concurrent-ruby-edge/concurrent/edge/version') +version = File.read("#{__dir__}/lib/concurrent-ruby/concurrent/version.rb")[/'(.+)'/, 1] or raise +edge_version = File.read("#{__dir__}/lib/concurrent-ruby-edge/concurrent/edge/version.rb")[/'(.+)'/, 1] or raise Gem::Specification.new do |s| git_files = `git ls-files`.split("\n") s.name = 'concurrent-ruby-edge' - s.version = Concurrent::EDGE_VERSION + s.version = edge_version s.platform = Gem::Platform::RUBY s.authors = ["Jerry D'Antonio", 'Petr Chalupa', 'The Ruby Concurrency Team'] s.email = 'concurrent-ruby@googlegroups.com' @@ -25,5 +25,5 @@ Please see http://concurrent-ruby.com for more information. s.required_ruby_version = '>= 2.3' - s.add_runtime_dependency 'concurrent-ruby', "~> #{Concurrent::VERSION.split('.')[0..1].join('.')}" + s.add_runtime_dependency 'concurrent-ruby', "~> #{version.split('.')[0..1].join('.')}" end diff --git a/concurrent-ruby-ext.gemspec b/concurrent-ruby-ext.gemspec index 534822de0..4bfafdd5b 100644 --- a/concurrent-ruby-ext.gemspec +++ b/concurrent-ruby-ext.gemspec @@ -1,8 +1,8 @@ -require File.join(File.dirname(__FILE__ ), 'lib/concurrent-ruby/concurrent/version') +version = File.read("#{__dir__}/lib/concurrent-ruby/concurrent/version.rb")[/'(.+)'/, 1] or raise Gem::Specification.new do |s| s.name = 'concurrent-ruby-ext' - s.version = Concurrent::VERSION + s.version = version s.platform = Gem::Platform::RUBY s.authors = ["Jerry D'Antonio", 'The Ruby Concurrency Team'] s.email = 'concurrent-ruby@googlegroups.com' @@ -23,5 +23,5 @@ Gem::Specification.new do |s| s.required_ruby_version = '>= 2.3' - s.add_runtime_dependency 'concurrent-ruby', "= #{Concurrent::VERSION}" + s.add_runtime_dependency 'concurrent-ruby', "= #{version}" end diff --git a/concurrent-ruby.gemspec b/concurrent-ruby.gemspec index 863201b54..1537d377d 100644 --- a/concurrent-ruby.gemspec +++ b/concurrent-ruby.gemspec @@ -1,10 +1,10 @@ -require File.join(File.dirname(__FILE__ ), 'lib/concurrent-ruby/concurrent/version') +version = File.read("#{__dir__}/lib/concurrent-ruby/concurrent/version.rb")[/'(.+)'/, 1] or raise Gem::Specification.new do |s| git_files = `git ls-files`.split("\n") s.name = 'concurrent-ruby' - s.version = Concurrent::VERSION + s.version = version s.platform = Gem::Platform::RUBY s.authors = ["Jerry D'Antonio", 'Petr Chalupa', 'The Ruby Concurrency Team'] s.email = 'concurrent-ruby@googlegroups.com' diff --git a/docs-source/signpost.md b/docs-source/signpost.md index 161812a8f..e48a56a16 100644 --- a/docs-source/signpost.md +++ b/docs-source/signpost.md @@ -3,7 +3,7 @@ Pick a `concurrent-ruby` version: * [master](./master/index.html) -* [1.3.1 with edge 0.7.0](./1.3.1/index.html) +* [1.3.4 with edge 0.7.1](./1.3.4/index.html) * [1.1.10 with edge 0.6.0](./1.1.10/index.html) * [1.1.9 with edge 0.6.0](./1.1.9/index.html) * [1.1.8 with edge 0.6.0](./1.1.8/index.html) diff --git a/lib/concurrent-ruby/concurrent/utility/processor_counter.rb b/lib/concurrent-ruby/concurrent/utility/processor_counter.rb index 6724457f1..2489cbd76 100644 --- a/lib/concurrent-ruby/concurrent/utility/processor_counter.rb +++ b/lib/concurrent-ruby/concurrent/utility/processor_counter.rb @@ -12,6 +12,7 @@ def initialize @processor_count = Delay.new { compute_processor_count } @physical_processor_count = Delay.new { compute_physical_processor_count } @cpu_quota = Delay.new { compute_cpu_quota } + @cpu_shares = Delay.new { compute_cpu_shares } end def processor_count @@ -41,6 +42,10 @@ def cpu_quota @cpu_quota.value end + def cpu_shares + @cpu_shares.value + end + private def compute_processor_count @@ -78,7 +83,7 @@ def compute_physical_processor_count # Bail out if both commands returned something unexpected processor_count else - # powershell: "\nNumberOfCores\n-------------\n 4\n\n\n" + # powershell: "\nNumberOfCores\n-------------\n 4\n\n\n" # wmic: "NumberOfCores \n\n4 \n\n\n\n" result.scan(/\d+/).map(&:to_i).reduce(:+) end @@ -107,12 +112,28 @@ def compute_cpu_quota elsif File.exist?("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us") # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt max = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").to_i - return nil if max == 0 + # If the cpu.cfs_quota_us is -1, cgroup does not adhere to any CPU time restrictions + # https://docs.kernel.org/scheduler/sched-bwc.html#management + return nil if max <= 0 period = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").to_f max / period end end end + + def compute_cpu_shares + if RbConfig::CONFIG["target_os"].include?("linux") + if File.exist?("/sys/fs/cgroup/cpu.weight") + # cgroups v2: https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files + # Ref: https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2 + weight = File.read("/sys/fs/cgroup/cpu.weight").to_f + ((((weight - 1) * 262142) / 9999) + 2) / 1024 + elsif File.exist?("/sys/fs/cgroup/cpu/cpu.shares") + # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt + File.read("/sys/fs/cgroup/cpu/cpu.shares").to_f / 1024 + end + end + end end end @@ -128,8 +149,8 @@ def compute_cpu_quota # `java.lang.Runtime.getRuntime.availableProcessors` will be used. According # to the Java documentation this "value may change during a particular # invocation of the virtual machine... [applications] should therefore - # occasionally poll this property." Subsequently the result will NOT be - # memoized under JRuby. + # occasionally poll this property." We still memoize this value once under + # JRuby. # # Otherwise Ruby's Etc.nprocessors will be used. # @@ -162,13 +183,14 @@ def self.physical_processor_count end # Number of processors cores available for process scheduling. - # Returns `nil` if there is no #cpu_quota, or a `Float` if the - # process is inside a cgroup with a dedicated CPU quota (typically Docker). + # This method takes in account the CPU quota if the process is inside a cgroup with a + # dedicated CPU quota (typically Docker). + # Otherwise it returns the same value as #processor_count but as a Float. # # For performance reasons the calculated value will be memoized on the first # call. # - # @return [nil, Float] number of available processors + # @return [Float] number of available processors def self.available_processor_count processor_counter.available_processor_count end @@ -187,4 +209,12 @@ def self.available_processor_count def self.cpu_quota processor_counter.cpu_quota end + + # The CPU shares requested by the process. For performance reasons the calculated + # value will be memoized on the first call. + # + # @return [Float, nil] CPU shares requested by the process, or nil if not set + def self.cpu_shares + processor_counter.cpu_shares + end end diff --git a/lib/concurrent-ruby/concurrent/version.rb b/lib/concurrent-ruby/concurrent/version.rb index 98e7f12b3..1b0c4c22f 100644 --- a/lib/concurrent-ruby/concurrent/version.rb +++ b/lib/concurrent-ruby/concurrent/version.rb @@ -1,3 +1,3 @@ module Concurrent - VERSION = '1.3.3' + VERSION = '1.3.4' end diff --git a/spec/concurrent/utility/processor_count_spec.rb b/spec/concurrent/utility/processor_count_spec.rb index fdc44b0ae..8de31e027 100644 --- a/spec/concurrent/utility/processor_count_spec.rb +++ b/spec/concurrent/utility/processor_count_spec.rb @@ -56,6 +56,15 @@ module Concurrent expect(counter.cpu_quota).to be_nil end + it 'returns nil if cgroups v1 and cpu.cfs_quota_us is -1' do + expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.max").and_return(false) + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").and_return(true) + + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").and_return("-1\n") + expect(counter.cpu_quota).to be_nil + end + it 'returns a float if cgroups v1 sets a limit' do expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.max").and_return(false) @@ -92,4 +101,26 @@ module Concurrent end end + + RSpec.describe '#cpu_shares' do + let(:counter) { Concurrent::Utility::ProcessorCounter.new } + + it 'returns a float when cgroups v2 sets a cpu.weight' do + expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.weight").and_return(true) + + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu.weight").and_return("10000\n") + expect(counter.cpu_shares).to be == 256.0 + end + + it 'returns a float if cgroups v1 sets a cpu.shares' do + expect(RbConfig::CONFIG).to receive(:[]).with("target_os").and_return("linux") + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu.weight").and_return(false) + expect(File).to receive(:exist?).with("/sys/fs/cgroup/cpu/cpu.shares").and_return(true) + + expect(File).to receive(:read).with("/sys/fs/cgroup/cpu/cpu.shares").and_return("512\n") + expect(counter.cpu_shares).to be == 0.5 + end + + end end